diff --git a/.github/workflows/nightly-test-npu-e2e-multi-node.yml b/.github/workflows/nightly-test-npu-e2e-multi-node.yml new file mode 100644 index 000000000..89b02ed6b --- /dev/null +++ b/.github/workflows/nightly-test-npu-e2e-multi-node.yml @@ -0,0 +1,232 @@ +name: 'Nightly Test (NPU) Multi Node Template' + +on: + workflow_call: + inputs: + runner: + required: false + type: string + default: linux-amd64-cpu-8 + test_type: + required: false + type: string + default: perf + description: perf or accuracy + test_config_name: + required: true + type: string + description: test config name + prefill_size: + required: false + type: number + default: 0 + description: number of prefill nodes for pd-separation + decode_size: + required: false + type: number + default: 0 + description: number of decode nodes for pd-separation + router_size: + required: false + type: number + default: 0 + description: number of router nodes for pd-separation + node_size: + required: false + type: number + default: 0 + description: number of nodes for pd-mix + test_case: + required: true + type: string + description: path of test case file + image: + required: false + type: string + description: image for pods + default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann8.5.0-a3" + install_sglang_from_source: + required: false + type: boolean + default: true + description: use sglang from source code or from docker image + prefill_decode_deployment: + required: true + type: string + description: "The deployment of prefill and decode nodes. ['separation', 'mix']" + transformers_version: + required: false + type: string + default: "" + description: "The transformers version number for running sglang. Use default version in image if keep empty." + +concurrency: + group: ascend-nightly-multi-node-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.test_config_name }} + cancel-in-progress: true + +jobs: + e2e: + name: ${{ inputs.test_config_name }} + runs-on: ${{ inputs.runner }} + container: + image: swr.ap-southeast-1.myhuaweicloud.com/base_image/ascend-ci/sglang:main-x86 + env: + KUBECONFIG: /root/.cache/.cache/kb.yaml + KUBECTL: /root/.cache/.cache/kubectl + NAMESPACE: sgl-project + KUBE_JOB_NAME: ascend-sglang-${{ inputs.test_type }}-test + SGLANG_IS_IN_CI: true + ASCEND_E2E_TEST_CONFIG_PATH: python/sglang/test/ascend/e2e + SGLANG_USE_MODELSCOPE: true + HF_ENDPOINT: https://hf-mirror.com + TRANSFORMERS_VERBOSITY: "error" + GDN_ATTN_BACKEND_TRITON: 1 + steps: + - name: Install dependencies + run: | + pip3 install -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple jinja2 + pip3 install -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple psutil + # Install kubernetes + pip3 install -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple kubernetes + cp $KUBECTL /usr/local/sbin/ + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Prepare code for testing + run: | + # copy source code to shared-disk + current_path=$(pwd) + target_path=/root/.cache/tests/sglang + rm -rf ${target_path} + mkdir -p ${target_path} + cp -r ${current_path}/* ${target_path}/ + + - name: Clear resources + run: | + cd ${ASCEND_E2E_TEST_CONFIG_PATH} + kubectl delete -f ./k8s_multi_pd_*.yaml --ignore-not-found=true || true + + pod_name_prefix="${KUBE_JOB_NAME}-" + echo "kube name space: $NAMESPACE, pod name prefix: ${pod_name_prefix}" + while true; do + if kubectl get po -A -n $NAMESPACE | grep -q "${pod_name_prefix}"; then + echo "Found exist sglang job, sleeping for 30 seconds..." + sleep 30 + kubectl get pods | grep "${pod_name_prefix}" | awk '{print $1}' | xargs kubectl delete pod -n $NAMESPACE || true + else + echo "No sglang job exist, start test case..." + break + fi + done + + - name: Run test + timeout-minutes: 300 + run: | + # sglang_source_relative_path is shared-disk path + sglang_source_relative_path=tests/sglang + sglang_source_path=/root/.cache/${sglang_source_relative_path} + echo "Source code path: ${sglang_source_path}" + + test_case="${{ inputs.test_case }}" + if [ ! -f "${sglang_source_path}/${test_case}" ]; then + echo "The testcase does not exit: ${sglang_source_path}/${test_case}" + exit 1 + fi + + # prepare for output data + current_date=$(date +%Y%m%d) + test_data_output_path=/root/.cache/tests/output/${{ inputs.test_type }}/${current_date} + mkdir -p ${test_data_output_path} + tc_name=${test_case##*/} + tc_name=${tc_name%.*} + metrics_data_path=${test_data_output_path}/${tc_name} + mkdir -p ${metrics_data_path} + echo "Metrics file path: ${metrics_data_path}" + + prefill_decode_deployment="${{ inputs.prefill_decode_deployment }}" + kube_job_type="" + if [ "$prefill_decode_deployment" = "separation" ];then + kube_job_type=multi-pd-separation + elif [ "$prefill_decode_deployment" = "mix" ];then + kube_job_type=multi-pd-mix + else + echo "Unsupported deployment type: ${prefill_decode_deployment}" + exit 1 + fi + + transformers_version="${{ inputs.transformers_version }}" + cd ${ASCEND_E2E_TEST_CONFIG_PATH} + CMD="python3 -u run_npu_e2e_test.py \ + --env ci \ + --image ${{ inputs.image }} \ + --sglang-source-relative-path ${sglang_source_relative_path} \ + --metrics-data-file ${metrics_data_path} \ + --test-case ${test_case} \ + --kube-name-space ${NAMESPACE} \ + --kube-job-type ${kube_job_type} \ + --kube-job-name-prefix ${KUBE_JOB_NAME}" + + if [ "$prefill_decode_deployment" = "separation" ];then + CMD="${CMD} \ + --prefill-size ${{ inputs.prefill_size }} \ + --decode-size ${{ inputs.decode_size }} \ + --router-size ${{ inputs.router_size }}" + elif [ "$prefill_decode_deployment" = "mix" ];then + CMD="${CMD} --node-size ${{ inputs.node_size }}" + else + echo "Unsupported deployment type: ${prefill_decode_deployment}" + exit 1 + fi + + if [ "$SGLANG_IS_IN_CI" = "true" ] || [ "$SGLANG_IS_IN_CI" = "True" ];then + echo "Run test in ci." + CMD="${CMD} --sglang-is-in-ci" + fi + + install_sglang_from_source="${{ inputs.install_sglang_from_source }}" + if [ "$install_sglang_from_source" = "true" ] || [ "$install_sglang_from_source" = "True" ];then + echo "Install sglang from source." + CMD="${CMD} --install-sglang-from-source" + commit_id=${{ github.sha }} + echo "commit id: ${commit_id}" > ${test_data_output_path}/commit_id + else + echo "Use sglang from image: ${{ inputs.image }}" + fi + + if [ "$transformers_version" != "" ];then + CMD="${CMD} --transformers-version ${transformers_version}" + else + echo "Use default transformers version in image." + fi + + echo "Run command: ${CMD}" + eval "${CMD}" + + - name: Upload metrics + if: always() + uses: actions/upload-artifact@v4 + with: + name: metrics-${{ inputs.test_config_name }} + path: /tmp/metrics.json + retention-days: 7 + + - name: Post process + if: always() + run: | + cd ${ASCEND_E2E_TEST_CONFIG_PATH} + kubectl get pods -n $NAMESPACE | grep $KUBE_JOB_NAME + kubectl delete -f ./k8s_multi_pd_*.yaml --ignore-not-found=true || true + + pod_name_prefix="${KUBE_JOB_NAME}-" + echo "kube name space: $NAMESPACE, pod name prefix: ${pod_name_prefix}" + while true; do + if kubectl get po -A -n $NAMESPACE | grep -q "${pod_name_prefix}"; then + echo "Found exist sglang job, sleeping for 30 seconds..." + sleep 30 + kubectl get pods | grep "${pod_name_prefix}" | awk '{print $1}' | xargs kubectl delete pod -n $NAMESPACE || true + else + echo "No sglang job exist, start test case..." + break + fi + done diff --git a/.github/workflows/nightly-test-npu-e2e-single-node.yml b/.github/workflows/nightly-test-npu-e2e-single-node.yml new file mode 100644 index 000000000..3d8ca0762 --- /dev/null +++ b/.github/workflows/nightly-test-npu-e2e-single-node.yml @@ -0,0 +1,228 @@ +name: 'Nightly Test (NPU) Single Node Template' + +on: + workflow_call: + inputs: + runner: + required: true + type: string + default: linux-aarch64-a3-16 + test_type: + required: false + type: string + default: perf + description: perf or accuracy + test_config_name: + required: true + type: string + description: test config name + test_case: + required: true + type: string + description: path of test case file + image: + required: false + type: string + description: image for pods + default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3" + install_sglang_from_source: + required: false + type: boolean + default: false + description: use sglang from source code or from docker image + transformers_version: + required: false + type: string + default: "" + description: "The transformers version number for running sglang. Use default version in image if keep empty." + +concurrency: + group: ascend-nightly-e2e-singlenode-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.test_config_name }} + cancel-in-progress: true + +jobs: + e2e: + name: ${{ inputs.test_config_name }} + runs-on: ${{ inputs.runner }} + container: + image: ${{ inputs.image }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Check npu info + run: | + npu-smi info + + - name: Run test + timeout-minutes: 120 + env: + SGLANG_USE_MODELSCOPE: true + HF_ENDPOINT: https://hf-mirror.com + SGLANG_IS_IN_CI: true + TRANSFORMERS_VERBOSITY: "error" + GDN_ATTN_BACKEND_TRITON: 1 + SGLANG_TEST_METRICS_OUTPUT: /root/.cache/tests/output/metrics/metrics + shell: bash + run: | + sglang_source_path=$(pwd) + echo "Source code path: ${sglang_source_path}" + ln -sf ${sglang_source_path} /root/sglang + + test_case=${{ inputs.test_case }} + if [ ! -f "${sglang_source_path}/${test_case}" ]; then + echo "The testcase does not exit: ${sglang_source_path}/${test_case}" + exit 1 + fi + + # prepare for output data + current_date=$(date +%Y%m%d) + test_data_output_path=/root/.cache/tests/output/${{ inputs.test_type }}/${current_date} + mkdir -p ${test_data_output_path} + tc_name=${test_case##*/} + tc_name=${tc_name%.*} + export METRICS_DATA_FILE=${test_data_output_path}/${tc_name} + mkdir -p ${METRICS_DATA_FILE} + echo "Metrics file path: ${METRICS_DATA_FILE}" + + # copy required file from our daily cache + cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp + curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl + + export TRANSFORMERS_VERSION_FOR_SGLANG="${{ inputs.transformers_version }}" + PYTHON_FOR_SGLANG="python" + PIP_FOR_SGLANG="pip" + if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then + echo "===== Install transformers for sglang - Begin =====" + TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG} + if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then + echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}." + echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} online." + pip install transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple + else + echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally." + TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG} + mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}" + cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/" + pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" + fi + echo "===== Install transformers for sglang in virtual env - End =====" + fi + echo "Transformers version for sglang: $(${PIP_FOR_SGLANG} show transformers | grep Version | cut -d: -f2)" + + echo "scaling_governor performance num: \ + $(cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor | grep performance | wc -l)" + echo "swappiness: $(cat /proc/sys/vm/swappiness)" + echo "numa_balancing: $(cat /proc/sys/kernel/numa_balancing)" + echo "sched_migration_cost_ns: $(cat /proc/sys/kernel/sched_migration_cost_ns)" + + export SGLANG_TEST_MAX_RETRY=0 + export SGLANG_SET_CPU_AFFINITY=1 + echo "SGLANG_SET_CPU_AFFINITY: $SGLANG_SET_CPU_AFFINITY" + + install_sglang_from_source=${{ inputs.install_sglang_from_source }} + if [ "$install_sglang_from_source" = "true" ] || [ "$install_sglang_from_source" = "True" ];then + echo "Install sglang from source" + commit_id=${{ github.sha }} + echo "commit id: ${commit_id}" > ${test_data_output_path}/commit_id + export PYTHONPATH=${sglang_source_path}/python:$PYTHONPATH + else + echo "Use sglang from image: ${{ inputs.image }}" + sglang_pkg_path=/sgl-workspace/sglang/python + ascend_test_util_path=${sglang_pkg_path}/sglang/test/ascend + mkdir -p ${ascend_test_util_path} + mv ${ascend_test_util_path} ${ascend_test_util_path}_bak + cp -r ${sglang_source_path}/python/sglang/test/ascend ${ascend_test_util_path} + fi + + source /usr/local/Ascend/cann/set_env.sh || true + source /usr/local/Ascend/nnal/atb/set_env.sh || true + + # Set environment of cann + log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}" + rm -rf ${log_path} + mkdir -p ${log_path} + echo "Log path: ${log_path}" + + echo "Running test case ${test_case}" + test_exit_code=0 + ${PYTHON_FOR_SGLANG} -u ${test_case} 2>&1 | tee /tmp/test_output.log || test_exit_code=$? + echo "Finished test case ${test_case}" + + if [ "${test_exit_code}" = "0" ]; then + test_status="pass" + status_icon="✅" + else + test_status="fail" + status_icon="❌" + fi + + echo "test_status=${test_status}" >> $GITHUB_ENV + echo "tc_name=${tc_name}" >> $GITHUB_ENV + export test_status tc_name + + echo "## ${tc_name} ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0) + if [ "${metric_count}" -gt 0 ]; then + echo "| Metric | Value | Pass |" >> $GITHUB_STEP_SUMMARY + echo "|--------|-------|------|" >> $GITHUB_STEP_SUMMARY + grep '\[METRIC\]' /tmp/test_output.log | while IFS= read -r line; do + metric_name=$(echo "$line" | sed -E 's/.*\[METRIC\] ([^=]+)=.*/\1/') + metric_value=$(echo "$line" | sed -E 's/.*\[METRIC\] [^=]+=([^ ]+).*/\1/') + echo "| ${metric_name} | ${metric_value} | ${status_icon} |" >> $GITHUB_STEP_SUMMARY + done + else + echo "No metrics collected (test may have failed before producing results)." >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY + + echo "import json, re, os" > /tmp/dump_metrics.py + echo "tc = os.environ.get('tc_name', 'unknown')" >> /tmp/dump_metrics.py + echo "st = os.environ.get('test_status', 'unknown')" >> /tmp/dump_metrics.py + echo "tp = '${{ inputs.test_type }}'" >> /tmp/dump_metrics.py + echo "metrics = {}" >> /tmp/dump_metrics.py + echo "with open('/tmp/test_output.log') as f:" >> /tmp/dump_metrics.py + echo " for line in f:" >> /tmp/dump_metrics.py + echo " m = re.match(r'\[METRIC\] (\S+)=(\S+)', line)" >> /tmp/dump_metrics.py + echo " if m:" >> /tmp/dump_metrics.py + echo " v = m.group(2)" >> /tmp/dump_metrics.py + echo " try:" >> /tmp/dump_metrics.py + echo " v = float(v)" >> /tmp/dump_metrics.py + echo " except ValueError:" >> /tmp/dump_metrics.py + echo " pass" >> /tmp/dump_metrics.py + echo " metrics[m.group(1)] = v" >> /tmp/dump_metrics.py + echo "baselines = {}" >> /tmp/dump_metrics.py + echo "for k, v in list(metrics.items()):" >> /tmp/dump_metrics.py + echo " if k.endswith('_baseline'):" >> /tmp/dump_metrics.py + echo " baselines[k[:-9]] = v" >> /tmp/dump_metrics.py + echo " del metrics[k]" >> /tmp/dump_metrics.py + echo "with open('/tmp/metrics.json', 'w') as f:" >> /tmp/dump_metrics.py + echo " json.dump({'test_case': tc, 'test_type': tp, 'status': st, 'metrics': metrics, 'baselines': baselines}, f)" >> /tmp/dump_metrics.py + python3 /tmp/dump_metrics.py + exit ${test_exit_code} + + - name: Upload metrics + if: always() + uses: actions/upload-artifact@v4 + with: + name: metrics-${{ inputs.test_config_name }} + path: /tmp/metrics.json + retention-days: 7 + + - name: Backup plog + if: always() + shell: bash + run: | + plog_path="/root/ascend/log/debug/plog" + if [ -d "$plog_path" ];then + echo "Plog files found. Begin to backup them." + tc_name=${{ inputs.test_case }} + tc_name=${tc_name##*/} + tc_name=${tc_name%.*} + target_plog_path="/root/.cache/tests/logs/plog/${tc_name}/${HOSTNAME}" + echo "Save path: ${target_plog_path}" + rm -rf ${target_plog_path} + mkdir -p ${target_plog_path} + cp ${plog_path}/* ${target_plog_path} + fi diff --git a/.github/workflows/nightly-test-npu.yml b/.github/workflows/nightly-test-npu.yml index 3ade42b0b..0e6f49ce3 100644 --- a/.github/workflows/nightly-test-npu.yml +++ b/.github/workflows/nightly-test-npu.yml @@ -1,5 +1,4 @@ name: Nightly Test (NPU) - on: schedule: - cron: '0 18 * * *' # Execute at 2:00 a.m. Beijing Time every day @@ -9,6 +8,32 @@ on: paths: - ".github/workflows/nightly-test-npu.yml" workflow_dispatch: + inputs: + ref: + description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' + required: false + type: string + default: '' + job_filter: + description: 'Select which job to run (leave empty or "all" to run all jobs)' + required: false + type: string + default: 'all' + image_a2: + description: 'The a2 running docker image of the test task.' + required: false + type: string + default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b' + image_a3: + description: 'The a3 running docker image of the test task.' + required: false + type: string + default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3' + skip_install_flag: + description: 'Indicates whether to skip the installation of sglang, defaulting to false.' + required: false + type: string + default: 'true' workflow_call: inputs: ref: @@ -21,28 +46,31 @@ on: required: false type: string default: 'all' + image_a2: + description: 'The a2 running docker image of the test task.' + required: false + type: string + default: '' image_a3: description: 'The a3 running docker image of the test task.' required: false type: string - default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11' + default: '' skip_install_flag: description: 'Indicates whether to skip the installation of sglang, defaulting to false.' required: false type: string - default: 'false' - - + default: '' concurrency: group: nightly-test-npu-${{ inputs.ref || github.ref }} cancel-in-progress: ${{ github.event_name != 'workflow_call' }} - jobs: set-image-config: runs-on: ubuntu-latest outputs: ref: ${{ steps.set-vars.outputs.ref }} job_filter: ${{ steps.set-vars.outputs.job_filter }} + image_a2: ${{ steps.set-vars.outputs.image_a2 }} image_a3: ${{ steps.set-vars.outputs.image_a3 }} skip_install_flag: ${{ steps.set-vars.outputs.skip_install_flag }} steps: @@ -55,379 +83,324 @@ jobs: else echo "ref=${{ inputs.ref }}" >> $GITHUB_OUTPUT fi - if [ -z "${{ inputs.job_filter }}" ]; then echo "job_filter=all" >> $GITHUB_OUTPUT else echo "job_filter=${{ inputs.job_filter }}" >> $GITHUB_OUTPUT fi - + if [ -z "${{ inputs.image_a2 }}" ]; then + echo "image_a2=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b" >> $GITHUB_OUTPUT + else + echo "image_a2=${{ inputs.image_a2 }}" >> $GITHUB_OUTPUT + fi if [ -z "${{ inputs.image_a3 }}" ]; then - echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT + echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3" >> $GITHUB_OUTPUT else echo "image_a3=${{ inputs.image_a3 }}" >> $GITHUB_OUTPUT fi - if [ -z "${{ inputs.skip_install_flag }}" ]; then echo "skip_install_flag=false" >> $GITHUB_OUTPUT else echo "skip_install_flag=${{ inputs.skip_install_flag }}" >> $GITHUB_OUTPUT fi - - nightly-1-npu-a3: - needs: [set-image-config] - if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} - runs-on: linux-aarch64-a3-2 + nightly-poc-single-node-a2-tests: + name: single-node-poc-a2 + if: ${{ !cancelled() }} + needs: [ set-image-config ] strategy: fail-fast: false + max-parallel: 6 matrix: - part: [0, 1] - container: - image: ${{ needs.set-image-config.outputs.image_a3 }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ needs.set-image-config.outputs.ref || github.ref }} + test_config: + - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2 + runner: linux-aarch64-a2-4 + test_case: test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py + test_type: 'perf' + uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml + with: + runner: ${{ matrix.test_config.runner }} + test_type: ${{ matrix.test_config.test_type }} + test_config_name: ${{ matrix.test_config.name }} + test_case: ${{ matrix.test_config.test_case }} + image: ${{ needs.set-image-config.outputs.image_a2 }} + install_sglang_from_source: false + transformers_version: '' - - name: Install dependencies - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - run: | - # speed up by using infra cache services - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" - - if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then - bash scripts/ci/npu/npu_ci_install_dependency.sh a3 - fi - - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - # copy gsm8k dataset - cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp - - - name: Print Log Information - run: | - bash scripts/ci/npu/npu_log_print.sh - - - name: Run test - timeout-minutes: 240 - env: - SGLANG_USE_MODELSCOPE: true - SGLANG_IS_IN_CI: true - HF_ENDPOINT: https://hf-mirror.com - TORCH_EXTENSIONS_DIR: /tmp/torch_extensions - PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" - STREAMS_PER_DEVICE: 32 - run: | - pip install sglang_router - hf download lmms-lab/MMMU --repo-type dataset - pip install sentence_transformers torchaudio==2.10.0 - pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap - pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv - git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git - cd ./lmms-eval - nohup pip install . > lmmslog.txt 2>&1 & - sleep 120 - export PYTHONPATH=$PYTHONPATH:$(pwd) - cd ../ - cd test - python3 run_suite.py --hw npu --suite nightly-1-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 - - nightly-2-npu-a3: + nightly-poc-single-node-tests: + name: single-node-poc + if: ${{ !cancelled() }} needs: [set-image-config] - if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} - runs-on: linux-aarch64-a3-2 strategy: fail-fast: false + max-parallel: 6 matrix: - part: [0] - container: + test_config: + # qwen3_6_35b_a3b performance tests + - name: qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py + test_type: 'perf' + - name: qwen3_6_35b_a3b_1p_aime26 + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py + - name: qwen3_6_35b_a3b_1p_in64k_out1k_50ms + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py + test_type: 'perf' + - name: qwen3_6_35b_a3b_1p_in128k_out1k_50ms + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py + test_type: 'perf' + - name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26 + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py + test_type: 'perf' + - name: qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py + test_type: 'perf' + # qwen3_6_27b performance tests + - name: qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py + test_type: 'perf' + - name: qwen3_6_27b_w8a8_2p_in16k_out1k_50ms + runner: linux-aarch64-a3-4 + test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py + test_type: 'perf' + - name: qwen3_6_27b_w8a8_2p_in64k_out1k_50ms + runner: linux-aarch64-a3-4 + test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py + test_type: 'perf' + - name: qwen3_6_27b_w8a8_2p_in128k_out1k_50ms + runner: linux-aarch64-a3-4 + test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py + test_type: 'perf' + - name: qwen3_6_27b_2p_in64k_out1k_prefix90_50ms + runner: linux-aarch64-a3-4 + test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py + test_type: 'perf' + - name: qwen3_6_27b_1p_in1024x1024_30_out1024_50ms + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py + test_type: 'perf' + - name: qwen3_6_27b_1p_in1080p_30_out256_50ms + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py + test_type: 'perf' + - name: qwen3_6_27b_1p_gpqa + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py + # qwen3_32b performance tests + - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa + runner: linux-aarch64-a3-4 + test_case: test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py + test_type: 'perf' + - name: qwen3_32b_bf16_8p_in18k_out4k_6ms + runner: linux-aarch64-a3-16 + test_case: test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py + test_type: 'perf' + - name: qwen3_32b_bf16_8p_gpqa + runner: linux-aarch64-a3-16 + test_case: test/registered/ascend/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py + # qwen3_30b_a3b performance tests + - name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25 + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py + test_type: 'perf' + # qwen3-8b performance tests + - name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py + test_type: 'perf' + - name: qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py + test_type: 'perf' + # qwen3_next_80b_a3b_instruct performance tests + - name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25 + runner: linux-aarch64-a3-4 + test_case: test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py + test_type: 'perf' + # minimax_m2_5 performance tests + - name: minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa + runner: linux-aarch64-a3-16 + test_case: test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py + test_type: 'perf' + - name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa + runner: linux-aarch64-a3-16 + test_case: test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py + test_type: 'perf' + # deepseek_v3_2 accuracy tests + - name: deepseek_v3_2_8p_aime25 + runner: linux-aarch64-a3-16 + test_case: test/registered/ascend/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py + # glm4_7_flash accuracy tests + - name: glm4_7_flash_1p_aime25 + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py + # glm4_6v_flash accuracy tests + - name: glm4_6v_flash_1p_mmmu + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py + # qwen3_vl_8b_thinking accuracy tests + - name: qwen3_vl_8b_thinking_1p_mmmu + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py + # qwen3_vl_30b_a3b_thinking accuracy tests + - name: qwen3_vl_30b_a3b_thinking_1p_mmmu + runner: linux-aarch64-a3-2 + test_case: test/registered/ascend/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py + uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml + with: + runner: ${{ matrix.test_config.runner }} + test_type: ${{ matrix.test_config.test_type }} + test_config_name: ${{ matrix.test_config.name }} + test_case: ${{ matrix.test_config.test_case }} image: ${{ needs.set-image-config.outputs.image_a3 }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ needs.set-image-config.outputs.ref || github.ref }} + install_sglang_from_source: false + transformers_version: '' - - name: Install dependencies - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - run: | - # speed up by using infra cache services - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" - - if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then - bash scripts/ci/npu/npu_ci_install_dependency.sh a3 - fi - - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - # copy gsm8k dataset - cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp - - - name: Print Log Information - run: | - bash scripts/ci/npu/npu_log_print.sh - - name: Run test - timeout-minutes: 240 - env: - SGLANG_USE_MODELSCOPE: true - SGLANG_IS_IN_CI: true - HF_ENDPOINT: https://hf-mirror.com - TORCH_EXTENSIONS_DIR: /tmp/torch_extensions - PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" - STREAMS_PER_DEVICE: 32 - run: | - pip install sglang_router - hf download lmms-lab/MMMU --repo-type dataset - pip install sentence_transformers torchaudio==2.10.0 - pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap - pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv - git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git - cd ./lmms-eval - nohup pip install . > lmmslog.txt 2>&1 & - sleep 120 - export PYTHONPATH=$PYTHONPATH:$(pwd) - cd ../ - cd test - python3 run_suite.py --hw npu --suite nightly-2-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 --auto-partition-id ${{ matrix.part }} --auto-partition-size 1 - - nightly-4-npu-a3: - needs: [set-image-config] - if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} - runs-on: linux-aarch64-a3-4 + nightly-poc-multi-node-tests: + name: multi-node-poc + if: ${{ !cancelled() }} + needs: [set-image-config, nightly-poc-single-node-tests] strategy: fail-fast: false + max-parallel: 1 matrix: - part: [0] - container: + test_config: + # glm5_1 performance tests + - name: glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26 + prefill_size: 2 + decode_size: 2 + router_size: 1 + test_case: test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py + test_type: 'perf' + prefill_decode_deployment: 'separation' + uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml + with: + runner: linux-amd64-cpu-8 + test_type: ${{ matrix.test_config.test_type }} + test_config_name: ${{ matrix.test_config.name }} + prefill_size: ${{ matrix.test_config.prefill_size }} + decode_size: ${{ matrix.test_config.decode_size }} + router_size: ${{ matrix.test_config.router_size }} + test_case: ${{ matrix.test_config.test_case }} image: ${{ needs.set-image-config.outputs.image_a3 }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ needs.set-image-config.outputs.ref|| github.ref }} + install_sglang_from_source: false + prefill_decode_deployment: ${{ matrix.test_config.prefill_decode_deployment }} + transformers_version: '' - - name: Install dependencies - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - run: | - # speed up by using infra cache services - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" - - if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then - bash scripts/ci/npu/npu_ci_install_dependency.sh a3 - fi - - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - # copy gsm8k dataset - cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp - - - name: Print Log Information - run: | - bash scripts/ci/npu/npu_log_print.sh - - - name: Run test - timeout-minutes: 240 - env: - SGLANG_USE_MODELSCOPE: true - SGLANG_IS_IN_CI: true - HF_ENDPOINT: https://hf-mirror.com - TORCH_EXTENSIONS_DIR: /tmp/torch_extensions - PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" - STREAMS_PER_DEVICE: 32 - run: | - pip install sglang_router - hf download lmms-lab/MMMU --repo-type dataset - pip install sentence_transformers torchaudio==2.10.0 - pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap - pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv - git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git - cd ./lmms-eval - nohup pip install . > lmmslog.txt 2>&1 & - sleep 120 - export PYTHONPATH=$PYTHONPATH:$(pwd) - cd ../ - cd test - python3 run_suite.py --hw npu --suite nightly-4-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 --auto-partition-id ${{ matrix.part }} --auto-partition-size 1 - - nightly-8-npu-a3: - needs: [set-image-config] - if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} - runs-on: linux-aarch64-a3-8 + nightly-poc-multi-node-mix-tests: + name: multi-node-mix-poc + if: ${{ !cancelled() }} + needs: [set-image-config, nightly-poc-single-node-tests, nightly-poc-multi-node-tests] strategy: fail-fast: false + max-parallel: 1 matrix: - part: [0] - container: + test_config: + # kimi_k2_6 performance tests + - name: kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25 + node_size: 2 + test_case: test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py + test_type: 'perf' + uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml + with: + runner: linux-amd64-cpu-8 + test_type: ${{ matrix.test_config.test_type }} + test_config_name: ${{ matrix.test_config.name }} + node_size: ${{ matrix.test_config.node_size }} + test_case: ${{ matrix.test_config.test_case }} image: ${{ needs.set-image-config.outputs.image_a3 }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ needs.set-image-config.outputs.ref || github.ref }} - - - name: Install dependencies - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - run: | - # speed up by using infra cache services - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" - - if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then - bash scripts/ci/npu/npu_ci_install_dependency.sh a3 - fi - - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - # copy gsm8k dataset - cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp - - - name: Print Log Information - run: | - bash scripts/ci/npu/npu_log_print.sh - - - name: Run test - timeout-minutes: 240 - env: - SGLANG_USE_MODELSCOPE: true - SGLANG_IS_IN_CI: true - HF_ENDPOINT: https://hf-mirror.com - TORCH_EXTENSIONS_DIR: /tmp/torch_extensions - PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" - STREAMS_PER_DEVICE: 32 - run: | - pip install sglang_router - hf download lmms-lab/MMMU --repo-type dataset - pip install sentence_transformers torchaudio==2.10.0 - pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap - pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv - git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git - cd ./lmms-eval - nohup pip install . > lmmslog.txt 2>&1 & - sleep 120 - export PYTHONPATH=$PYTHONPATH:$(pwd) - cd ../ - cd test - python3 run_suite.py --hw npu --suite nightly-8-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 --auto-partition-id ${{ matrix.part }} --auto-partition-size 1 - - nightly-16-npu-a3: - needs: [set-image-config] - if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }} - runs-on: linux-aarch64-a3-16 - strategy: - fail-fast: false - matrix: - part: [0, 1] - container: - image: ${{ needs.set-image-config.outputs.image_a3 }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ needs.set-image-config.outputs.ref || github.ref }} - - - name: Install dependencies - env: - TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu" - PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" - GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/" - run: | - # speed up by using infra cache services - CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" - sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list - pip config set global.index-url http://${CACHING_URL}/pypi/simple - pip config set global.trusted-host "${CACHING_URL}" - - if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then - bash scripts/ci/npu/npu_ci_install_dependency.sh a3 - fi - - # copy required file from our daily cache - cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp - # copy gsm8k dataset - cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp - - - name: Print Log Information - run: | - bash scripts/ci/npu/npu_log_print.sh - - - name: Run test - timeout-minutes: 240 - env: - SGLANG_USE_MODELSCOPE: true - SGLANG_IS_IN_CI: true - HF_ENDPOINT: https://hf-mirror.com - TORCH_EXTENSIONS_DIR: /tmp/torch_extensions - PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True" - STREAMS_PER_DEVICE: 32 - run: | - pip install sglang_router - hf download lmms-lab/MMMU --repo-type dataset - pip install sentence_transformers torchaudio==2.10.0 - pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap - pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv - git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git - cd ./lmms-eval - nohup pip install . > lmmslog.txt 2>&1 & - sleep 120 - export PYTHONPATH=$PYTHONPATH:$(pwd) - cd ../ - cd test - python3 run_suite.py --hw npu --suite nightly-16-npu-a3 --nightly --continue-on-error --timeout-per-file 3600 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 - + install_sglang_from_source: false + prefill_decode_deployment: 'mix' + transformers_version: '' check-all-jobs: - if: github.repository == 'sgl-project/sglang' && always() + if: ${{ !cancelled() }} needs: - - nightly-1-npu-a3 - - nightly-2-npu-a3 - - nightly-4-npu-a3 - - nightly-8-npu-a3 - - nightly-16-npu-a3 + - nightly-poc-single-node-a2-tests + - nightly-poc-single-node-tests + - nightly-poc-multi-node-tests + - nightly-poc-multi-node-mix-tests runs-on: ubuntu-latest - container: - image: docker.m.daocloud.io/ubuntu:22.04 steps: - - name: Check if any job failed + - name: Download all metrics + uses: actions/download-artifact@v4 + with: + pattern: metrics-* + path: /tmp/metrics + merge-multiple: false + + - name: Generate results table run: | - if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then - echo "One or more nightly test jobs failed" - exit 1 + status_emoji() { + case "$1" in + pass) echo "✅" ;; + fail) echo "❌" ;; + *) echo "❓" ;; + esac + } + + single_result_a2="${{ needs.nightly-poc-single-node-a2-tests.result }}" + single_result="${{ needs.nightly-poc-single-node-tests.result }}" + multi_result="${{ needs.nightly-poc-multi-node-tests.result }}" + mix_result="${{ needs.nightly-poc-multi-node-mix-tests.result }}" + + group_icon() { + case "$1" in + success) echo "✅" ;; + failure) echo "❌" ;; + cancelled) echo "⏭️" ;; + skipped) echo "⏭️" ;; + *) echo "❓" ;; + esac + } + + echo "## Nightly Test Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Group | Status |" >> $GITHUB_STEP_SUMMARY + echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY + echo "| single-node-poc-a2 | $(group_icon ${single_result_a2}) ${single_result_a2} |" >> $GITHUB_STEP_SUMMARY + echo "| single-node-poc | $(group_icon ${single_result}) ${single_result} |" >> $GITHUB_STEP_SUMMARY + echo "| multi-node-poc | $(group_icon ${multi_result}) ${multi_result} |" >> $GITHUB_STEP_SUMMARY + echo "| multi-node-mix-poc | $(group_icon ${mix_result}) ${mix_result} |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "## Per-Test Metrics" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + has_metrics=false + for dir in /tmp/metrics/metrics-*/; do + file="${dir}metrics.json" + if [ -f "$file" ]; then + has_metrics=true + echo "import json" > /tmp/parse_metrics.py + echo "with open('$file') as f:" >> /tmp/parse_metrics.py + echo " d = json.load(f)" >> /tmp/parse_metrics.py + echo "tc = d.get('test_case', '-')" >> /tmp/parse_metrics.py + echo "tp = d.get('test_type', '-')" >> /tmp/parse_metrics.py + echo "st = d.get('status', 'unknown')" >> /tmp/parse_metrics.py + echo "icon = '✅' if st == 'pass' else '❌'" >> /tmp/parse_metrics.py + echo "metrics = d.get('metrics', {})" >> /tmp/parse_metrics.py + echo "baselines = d.get('baselines', {})" >> /tmp/parse_metrics.py + echo "mstr = ', '.join(f'{k}={v}' for k,v in metrics.items()) if metrics else '-'" >> /tmp/parse_metrics.py + echo "bstr = ', '.join(f'{k}={v}' for k,v in baselines.items()) if baselines else '-'" >> /tmp/parse_metrics.py + echo "print(f'| {tc} | {tp} | {icon} {st} | {mstr} | {bstr} |')" >> /tmp/parse_metrics.py + python3 /tmp/parse_metrics.py + fi + done > /tmp/metrics_table.txt + + if [ "${has_metrics}" = "true" ]; then + echo "| Test Case | Type | Status | Metrics | Baseline |" >> $GITHUB_STEP_SUMMARY + echo "|-----------|------|--------|---------|----------|" >> $GITHUB_STEP_SUMMARY + cat /tmp/metrics_table.txt >> $GITHUB_STEP_SUMMARY + else + echo "No per-test metrics available (artifacts not found)." >> $GITHUB_STEP_SUMMARY fi - if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then - echo "One or more nightly test jobs were cancelled" - exit 1 - fi - echo "All nightly test jobs passed" + echo "" >> $GITHUB_STEP_SUMMARY + + FAIL=0 + if [ "${single_result}" != "success" ] && [ "${single_result}" != "skipped" ]; then FAIL=1; fi + exit $FAIL diff --git a/python/sglang/test/ascend/disaggregation_utils.py b/python/sglang/test/ascend/disaggregation_utils.py new file mode 100644 index 000000000..cb46a9d3b --- /dev/null +++ b/python/sglang/test/ascend/disaggregation_utils.py @@ -0,0 +1,153 @@ +import logging +import os +import time +import warnings +from urllib.parse import urlparse + +import requests + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_with_error_check, +) + +logger = logging.getLogger(__name__) + + +class TestDisaggregationBase(CustomTestCase): + @classmethod + def setUpClass(cls): + parsed_url = urlparse(DEFAULT_URL_FOR_TEST) + cls.base_host = parsed_url.hostname + base_port = str(parsed_url.port) + cls.lb_port = base_port + cls.prefill_port = f"{int(base_port) + 100}" + cls.decode_port = f"{int(base_port) + 200}" + cls.prefill_url = f"http://{cls.base_host}:{cls.prefill_port}" + cls.decode_url = f"http://{cls.base_host}:{cls.decode_port}" + cls.lb_url = f"http://{cls.base_host}:{cls.lb_port}" + print(f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=}") + cls.process_lb, cls.process_decode, cls.process_prefill = None, None, None + + # config transfer backend and rdma devices + cls.transfer_backend = [ + "--disaggregation-transfer-backend", + envs.SGLANG_TEST_PD_DISAGG_BACKEND.get(), + ] + cls.rdma_devices = [ + "--disaggregation-ib-device", + envs.SGLANG_TEST_PD_DISAGG_DEVICES.get(), + ] + if cls.rdma_devices[1] is None: + cls.rdma_devices = [] + msg = "No RDMA devices specified for disaggregation test, using default settings." + warnings.warn(msg) + + @classmethod + def launch_lb(cls): + lb_command = [ + "python3", + "-m", + "sglang_router.launch_router", + "--pd-disaggregation", + "--mini-lb", + "--prefill", + cls.prefill_url, + "--decode", + cls.decode_url, + "--host", + cls.base_host, + "--port", + cls.lb_port, + ] + print("Starting load balancer:", " ".join(lb_command)) + cls.process_lb = popen_with_error_check(lb_command) + cls.wait_server_ready(cls.lb_url + "/health") + + @classmethod + def wait_server_ready(cls, url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH): + start_time = time.perf_counter() + while True: + try: + response = requests.get(url) + if response.status_code == 200: + print(f"Server {url} is ready") + return + except Exception: + pass + + if time.perf_counter() - start_time > timeout: + raise RuntimeError(f"Server {url} failed to start in {timeout}s") + time.sleep(1) + + @classmethod + def tearDownClass(cls): + for process in [cls.process_lb, cls.process_decode, cls.process_prefill]: + if process: + try: + kill_process_tree(process.pid) + except Exception as e: + print(f"Error killing process {process.pid}: {e}") + + # wait for 5 seconds + time.sleep(5) + + +def get_rdma_devices_args(): + def _parse_list_env(var_name: str): + val = os.getenv(var_name) + if not val: + return None + items = [x.strip() for x in val.split(",") if x.strip()] + return items or None + + def _pick_default_pair(rdma_all_devices): + return [rdma_all_devices[0], rdma_all_devices[len(rdma_all_devices) // 2]] + + rdma_all_devices = _parse_list_env("SGLANG_CI_RDMA_ALL_DEVICES") or [ + f"mlx5_roce{i}" for i in range(8) + ] + logger.info("Resolved rdma_all_devices=%s", rdma_all_devices) + + n_rdma = len(rdma_all_devices) + + # 1. Get visible GPU indices + cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES") + if not cuda_visible_devices: + warnings.warn("CUDA_VISIBLE_DEVICES is not set. Using default RDMA devices.") + return ",".join(_pick_default_pair(rdma_all_devices)) + + try: + # Convert to list of integers (handling possible spaces and empty strings) + gpu_indices = [ + int(idx.strip()) for idx in cuda_visible_devices.split(",") if idx.strip() + ] + if not gpu_indices or len(gpu_indices) > 4: + return ",".join(_pick_default_pair(rdma_all_devices)) + except ValueError: + warnings.warn(f"Invalid CUDA_VISIBLE_DEVICES format: {cuda_visible_devices}") + return ",".join(_pick_default_pair(rdma_all_devices)) + + # 2. Calculate base RDMA index group (each group of 4 GPUs uses consecutive devices) + base_rdma_group = (min(gpu_indices) // 4) * 4 + for gpu_idx in gpu_indices: + if not (base_rdma_group <= gpu_idx < base_rdma_group + 4): + warnings.warn( + f"GPU index {gpu_idx} is outside expected group " + f"{base_rdma_group}-{base_rdma_group+3}" + ) + + # 3. Generate RDMA device names + rdma_devices = [] + for gpu_idx in gpu_indices: + nic_index = gpu_idx // (8 // n_rdma) + rdma_devices.append(rdma_all_devices[nic_index]) + + if not rdma_devices: + return ",".join(_pick_default_pair(rdma_all_devices)) + + return ",".join(rdma_devices) diff --git a/python/sglang/test/ascend/e2e/__init__.py b/python/sglang/test/ascend/e2e/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/sglang/test/ascend/e2e/gen_dataset_fixed_len.py b/python/sglang/test/ascend/e2e/gen_dataset_fixed_len.py new file mode 100644 index 000000000..f3bddc594 --- /dev/null +++ b/python/sglang/test/ascend/e2e/gen_dataset_fixed_len.py @@ -0,0 +1,521 @@ +import json +import os +import random +import string + +import numpy as np +from PIL import Image +from transformers import AutoTokenizer + + +def load_jsonl(path): + """Load data from a JSONL file, one JSON object per line.""" + data = [] + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + data.append(json.loads(line)) + return data + + +def save_jsonl(data, file_path): + """Save a list of dicts to a JSONL file, one JSON object per line.""" + file_dir = os.path.dirname(file_path) + if file_dir: + os.makedirs(file_dir, exist_ok=True) + with open(file_path, "w", encoding="utf-8") as f: + for item in data: + f.write(json.dumps(item, ensure_ascii=False) + "\n") + + +def format_qa(item): + """Format a GSM8K data entry into QA text for the few-shot pool.""" + question = item["question"] + answer = item["answer"] + return f"Question: {question}\nLet's think step by step\nAnswer:\n{answer}\n\n" + + +def pad_to_target_tokens( + question, + few_shot_pool_token_ids, + tokenizer, + target_tokens, + test_template="Question: {question}\nLet's think step by step\nAnswer:\n", +): + """Pad a question text to the target token length. + + Tokenizes the question using the test_template, calculates the remaining tokens + needed, and prepends randomly sampled few-shot token ids from the pool to reach + target_tokens. If the few-shot pool is insufficient, repeats the first sample + to fill the remaining gap. + + Args: + question: The test question text. + few_shot_pool_token_ids: List of token id lists from the few-shot training pool. + tokenizer: The tokenizer instance. + target_tokens: Target input token length. + test_template: Question template string, defaults to GSM8K format. + """ + test_prompt = test_template.format(question=question) + test_token_ids = tokenizer.encode(test_prompt, add_special_tokens=False) + + remaining_tokens = target_tokens - len(test_token_ids) + if remaining_tokens <= 0: + return tokenizer.decode( + test_token_ids[:target_tokens], skip_special_tokens=True + ) + + shuffled_ids = list(range(len(few_shot_pool_token_ids))) + random.shuffle(shuffled_ids) + + prefix_ids = [] + for idx in shuffled_ids: + fs_ids = few_shot_pool_token_ids[idx] + if len(prefix_ids) + len(fs_ids) <= remaining_tokens: + prefix_ids.extend(fs_ids) + else: + partial_gap = remaining_tokens - len(prefix_ids) + if partial_gap > 0: + prefix_ids.extend(fs_ids[:partial_gap]) + break + + if len(prefix_ids) < remaining_tokens and few_shot_pool_token_ids: + padding_source_ids = few_shot_pool_token_ids[shuffled_ids[0]] + repeat_count = (remaining_tokens // len(padding_source_ids)) + 1 + padding_ids = (padding_source_ids * repeat_count)[ + : remaining_tokens - len(prefix_ids) + ] + prefix_ids.extend(padding_ids) + + full_ids = prefix_ids + test_token_ids + return tokenizer.decode(full_ids[:target_tokens], skip_special_tokens=True) + + +def generate_custom_dataset( + train_path, + test_path, + tokenizer_path, + target_tokens, + num_prompts, + trust_remote_code=False, + test_template="Question: {question}\nLet's think step by step\nAnswer:\n", +): + """Generate a custom dataset with a fixed input token length. + + Builds a few-shot pool from the training set and pads test questions to the + specified token length. If the test set has fewer samples than num_prompts, + it cycles and repeats to fill the required count. + + Args: + train_path: Path to the GSM8K training JSONL file. + test_path: Path to the GSM8K test JSONL file. + tokenizer_path: Path to the tokenizer. + target_tokens: Target input token length. + num_prompts: Number of prompts to generate; 0 means use all test samples. + trust_remote_code: Whether to trust remote code when loading the tokenizer. + test_template: Question template string. + + Returns: + list[dict]: Each item contains fields defined in test_template. + """ + tokenizer = AutoTokenizer.from_pretrained( + tokenizer_path, trust_remote_code=trust_remote_code + ) + + train_data = load_jsonl(train_path) + test_data = load_jsonl(test_path) + if num_prompts > 0 and num_prompts > len(test_data): + multiplier = (num_prompts // len(test_data)) + 1 + test_data = (test_data * multiplier)[:num_prompts] + elif num_prompts > 0: + test_data = test_data[:num_prompts] + + few_shot_pool = [format_qa(item) for item in train_data] + few_shot_pool_token_ids = [ + tokenizer.encode(fs, add_special_tokens=False) for fs in few_shot_pool + ] + + output_data = [] + for i, test_item in enumerate(test_data): + padded_question = pad_to_target_tokens( + question=test_item["question"], + few_shot_pool_token_ids=few_shot_pool_token_ids, + tokenizer=tokenizer, + target_tokens=target_tokens, + test_template=test_template, + ) + output_data.append( + { + "question": padded_question, + "answer": test_item["answer"], + } + ) + if (i + 1) % 100 == 0: + actual_tokens = len( + tokenizer.encode(padded_question, add_special_tokens=False) + ) + print( + f"Processed {i + 1}/{len(test_data)}, last item tokens: {actual_tokens}" + ) + + token_counts = [ + len(tokenizer.encode(item["question"], add_special_tokens=False)) + for item in output_data + ] + print( + f"Token count stats: min={min(token_counts)}, max={max(token_counts)}, avg={sum(token_counts)/len(token_counts):.1f}" + ) + + return output_data + + +def generate_random_images(mm_dataset_data, size): + """Generate random image files for a multimodal dataset. + + Creates random RGB images at the specified resolution for each image path + listed in the dataset entries. + + Args: + mm_dataset_data: List of multimodal data entries, each with a "path" field + containing a list of image file paths. + size: Image size tuple (width, height), e.g. (1080, 1920). + """ + total_image_num = len(mm_dataset_data) + print(f"begin to generate images, total {total_image_num}") + + file_count = 0 + for item in mm_dataset_data: + image_paths = item.get("path") + + for image_path in image_paths: + if not image_path: + print("Error: The image path is none.") + continue + + dir_name = os.path.dirname(image_path) + if dir_name and not os.path.exists(dir_name): + os.makedirs(dir_name, exist_ok=True) + + random_array = np.random.randint( + 0, 256, (size[1], size[0], 3), dtype=np.uint8 + ) + + img = Image.fromarray(random_array) + img.save(image_path, quality=95) + if os.path.isfile(image_path): + file_count += 1 + + print(f"Finish images generation. Image num: {file_count}") + + +def generate_mm_dataset( + train_path, + test_path, + tokenizer_path, + target_tokens=3500, + num_prompts=1024, + trust_remote_code=False, + test_template="Question: {question}\nLet's think step by step\nAnswer:\n", + image_dir="/tmp/datasets/image", + size=None, +): + """Generate a multimodal (text + image) dataset. + + First generates fixed-length text data via generate_fixed_len_dataset, then + attaches random image paths and type labels to each entry, and generates + the corresponding random image files. + + Args: + train_path: Path to the GSM8K training JSONL file. + test_path: Path to the GSM8K test JSONL file. + tokenizer_path: Path to the tokenizer. + target_tokens: Target input token length. + num_prompts: Number of prompts to generate. + trust_remote_code: Whether to trust remote code when loading the tokenizer. + test_template: Question template string. + image_dir: Directory to save generated image files. + size: Image size string in "widthxheight" format, e.g. "1080x1920". + + Returns: + list[dict]: Each item contains "question", "answer", "type", and "path" fields. + """ + output_data = [] + text_data = generate_custom_dataset( + train_path, + test_path, + tokenizer_path, + target_tokens, + num_prompts, + trust_remote_code, + test_template, + ) + + for item in text_data: + random_string = "".join( + random.choices(string.ascii_letters + string.digits, k=10) + ) + item["type"] = "image" + item["path"] = [f"{image_dir}/{random_string}.jpg"] + output_data.append(item) + + size = tuple(map(int, size.split("x"))) + generate_random_images(output_data, size) + return output_data + + +def generate_gsm8k_dataset( + model_path, source_dataset_path, batch_size, input_len, output_file +): + """Generate a dataset with a fixed input token length from GSM8K (JSONL format). + + Reads GSM8K source data, repeats or truncates each question's tokens to input_len, + then trims or replicates the dataset to batch_size entries, shuffles, and writes + to the output file. + + Args: + model_path: Model path used to load the tokenizer. + source_dataset_path: Path to the GSM8K source JSONL file. + batch_size: Number of samples to generate. + input_len: Target input token length. + output_file: Output JSONL file path. + """ + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + dataset = [] + with open(source_dataset_path, "r", encoding="utf-8") as f: + for line in f: + data = json.loads(line) + dataset.append(data["question"]) + + dataset_new = [] + for sentence in dataset: + words = tokenizer.tokenize(sentence) + len_num = len(words) // input_len + if len_num == 0: + multiplier = (input_len // len(words)) + 1 + repeated_len = words * multiplier + words = repeated_len[:input_len] + decoded_text = tokenizer.convert_tokens_to_string(words) + if len(words) != input_len: + print( + f"Generate DataSet Error: the length of new input is {len(words)}, not {input_len}" + ) + dataset_new.append(decoded_text) + + batch_num = len(dataset_new) // batch_size + if batch_num == 0: + multiplier = (batch_size // len(dataset_new)) + 1 + repeated_batch = dataset_new * multiplier + dataset_new = repeated_batch[:batch_size] + else: + dataset_new = dataset_new[:batch_size] + + random.shuffle(dataset_new) + + if len(dataset_new) != batch_size: + print( + f"Generate DataSet Error: the size of new dataset is {len(dataset_new)}, not {batch_size}" + ) + + output_dir = os.path.dirname(output_file) + if output_dir: + os.makedirs(output_dir, exist_ok=True) + + with open(output_file, "w", encoding="utf-8") as f: + for i in range(len(dataset_new)): + f.write( + json.dumps( + {"question": f"{dataset_new[i]}", "answer": "none"}, + ensure_ascii=False, + ) + ) + f.write("\n") + + +def generate_random_dataset( + model_path, + source_dataset_path, + batch_size, + input_len, + output_file, + output_len=1024, + range_ratio=1, +): + """Generate a random dataset with logic matching bench_serving's --dataset-name random. + + Samples real conversation text from the ShareGPT dataset as prompts, adjusting + to the target token length via truncation or repetition. Input/output lengths + are randomly sampled from [target*range_ratio, target]. Output format is a + JSON array compatible with ais_bench's ShareGPTDataset. + + If source_dataset_path is not a valid JSON file, automatically downloads the + ShareGPT dataset from HuggingFace (anon8231489123/ShareGPT_Vicuna_unfiltered). + + Args: + model_path: Model path used to load the tokenizer. + source_dataset_path: Path to the ShareGPT JSON file; auto-downloaded if invalid. + batch_size: Number of samples to generate. + input_len: Target input token length. + output_file: Output JSON file path. + output_len: Target output token length, default 1024. + range_ratio: Random range ratio for input/output lengths. Actual lengths are + uniformly sampled from [target*range_ratio, target]. Default 1 (fixed length). + """ + SHAREGPT_REPO_ID = "anon8231489123/ShareGPT_Vicuna_unfiltered" + SHAREGPT_FILENAME = "ShareGPT_V3_unfiltered_cleaned_split.json" + + def _is_file_valid_json(path): + """Check if the path points to a valid JSON file (exists and parseable).""" + if not os.path.isfile(path): + return False + try: + with open(path, encoding="utf-8") as f: + json.load(f) + return True + except json.JSONDecodeError: + return False + + def _download_and_cache_hf_file(repo_id, filename, repo_type="dataset"): + """Download and cache a file from HuggingFace Hub.""" + from huggingface_hub import hf_hub_download + + return hf_hub_download(repo_id=repo_id, filename=filename, repo_type=repo_type) + + tokenizer = AutoTokenizer.from_pretrained(model_path) + + # Randomly sample input/output lengths per request in [target*range_ratio, target] + input_lens = np.random.randint( + max(int(input_len * range_ratio), 1), + input_len + 1, + size=batch_size, + ).tolist() + output_lens = np.random.randint( + max(int(output_len * range_ratio), 1), + output_len + 1, + size=batch_size, + ).tolist() + + # Subtract special tokens to ensure the actual encoded length does not exceed target + num_special_tokens = int(tokenizer.num_special_tokens_to_add()) + for i in range(batch_size): + input_lens[i] = max(1, input_lens[i] - num_special_tokens) + + # Auto-download ShareGPT dataset from HuggingFace if local file is invalid + if not _is_file_valid_json(source_dataset_path): + print( + f"source_dataset_path '{source_dataset_path}' is not a valid file, downloading from HuggingFace..." + ) + source_dataset_path = _download_and_cache_hf_file( + repo_id=SHAREGPT_REPO_ID, + filename=SHAREGPT_FILENAME, + ) + + # Load ShareGPT dataset, filter for >=2 turns, take the first turn (human) as prompt + with open(source_dataset_path, "r", encoding="utf-8") as f: + dataset = json.load(f) + + dataset = [ + data + for data in dataset + if len(data.get("conversations", data.get("conversation", []))) >= 2 + ] + dataset = [ + ( + data.get("conversations", data.get("conversation", []))[0]["value"], + data.get("conversations", data.get("conversation", []))[1]["value"], + ) + for data in dataset + ] + random.shuffle(dataset) + + # Sample prompts, truncating or repeating tokens to reach target input length + input_requests = [] + for data in dataset: + i = len(input_requests) + if i == batch_size: + break + + prompt = data[0] + prompt_token_ids = tokenizer.encode(prompt) + prompt_len = len(prompt_token_ids) + + if prompt_len == 0: + continue + + if prompt_len > input_lens[i]: + input_ids = prompt_token_ids[: input_lens[i]] + else: + ratio = (input_lens[i] + prompt_len - 1) // prompt_len + input_ids = (prompt_token_ids * ratio)[: input_lens[i]] + input_content = tokenizer.decode(input_ids) + # Output format compatible with ais_bench ShareGPTDataset + input_requests.append( + { + "id": str(i), + "conversations": [ + {"from": "human", "value": input_content}, + {"from": "gpt", "value": "none"}, + ], + } + ) + + print(f"#Input tokens: {np.sum(input_lens[:len(input_requests)])}") + print(f"#Output tokens: {np.sum(output_lens[:len(input_requests)])}") + + output_dir = os.path.dirname(output_file) + if output_dir: + os.makedirs(output_dir, exist_ok=True) + + # Output as JSON array format, compatible with ais_bench's json.load() + with open(output_file, "w", encoding="utf-8") as f: + json.dump(input_requests, f, ensure_ascii=False, indent=2) + + +def main(): + import argparse + + parser = argparse.ArgumentParser( + description="Generate GSM8K dataset with exact input token length" + ) + parser.add_argument( + "--train_path", type=str, required=True, help="Path to GSM8K train.jsonl" + ) + parser.add_argument( + "--test_path", type=str, required=True, help="Path to GSM8K test.jsonl" + ) + parser.add_argument( + "--output_path", type=str, required=True, help="Output jsonl path" + ) + parser.add_argument( + "--tokenizer_path", type=str, required=True, help="Path to model tokenizer" + ) + parser.add_argument( + "--target_tokens", type=int, default=3500, help="Target input token length" + ) + parser.add_argument( + "--trust_remote_code", + action="store_true", + help="Trust remote code for tokenizer", + ) + parser.add_argument( + "--num_prompts", + type=int, + default=0, + help="Number of prompts to generate, 0 means all", + ) + args = parser.parse_args() + + output_data = generate_custom_dataset( + train_path=args.train_path, + test_path=args.test_path, + tokenizer_path=args.tokenizer_path, + target_tokens=args.target_tokens, + num_prompts=args.num_prompts, + trust_remote_code=args.trust_remote_code, + ) + save_jsonl(output_data, args.output_path) + print(f"Done! Output {len(output_data)} items to {args.output_path}") + + +if __name__ == "__main__": + main() diff --git a/python/sglang/test/ascend/e2e/k8s_multi_pd_mix.yaml.jinja2 b/python/sglang/test/ascend/e2e/k8s_multi_pd_mix.yaml.jinja2 new file mode 100644 index 000000000..1c58d7457 --- /dev/null +++ b/python/sglang/test/ascend/e2e/k8s_multi_pd_mix.yaml.jinja2 @@ -0,0 +1,134 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ kube_config_map }} + namespace: {{ name_space }} +data: {} + +--- +apiVersion: batch.volcano.sh/v1alpha1 +kind: Job +metadata: + name: {{ kube_job_name }} + namespace: {{ name_space }} + labels: + ring-controller.atlas: ascend-1980 + fault-scheduling: "force" +spec: + minAvailable: {{ node_size }} + schedulerName: volcano + policies: + - event: PodEvicted + action: RestartJob + queue: default + tasks: + - name: "sglang-node" + replicas: {{ node_size }} + template: + metadata: + labels: + app: sgl-ascend + ring-controller.atlas: ascend-1980 + spec: + hostNetwork: True + containers: + - image: {{ image }} + imagePullPolicy: Always + securityContext: + privileged: true + name: sgl-ascend + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: INSTALL_SGLANG_FROM_SOURCE + value: "{{ install_sglang_from_source }}" + - name: METRICS_DATA_FILE + value: "{{ metrics_data_file }}" + - name: KUBECONFIG + value: "{{ kube_config }}" + - name: NAMESPACE + value: "{{ name_space }}" + - name: KUBE_CONFIG_MAP + value: "{{ kube_config_map }}" + - name: HF_ENDPOINT + value: "https://hf-mirror.com" + - name: SGLANG_IS_IN_CI + value: "{{ sglang_is_in_ci }}" + command: ["/bin/bash", "-c"] + args: + - | + {% if env in ["ci", "debug"] %} + bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }} + {% endif %} + {% if env == "green" %} + bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase_green.sh {{ test_case }} + {% endif %} + resources: + requests: + {% if env in ["ci", "debug"] %} + huawei.com/ascend-1980: 16 + {% endif %} + {% if env == "green" %} + huawei.com/Ascend910: 16 + {% endif %} + limits: + {% if env in ["ci", "debug"] %} + huawei.com/ascend-1980: 16 + {% endif %} + {% if env == "green" %} + huawei.com/Ascend910: 16 + {% endif %} + volumeMounts: + - name: ascend-driver + mountPath: /usr/local/Ascend/driver + - name: localtime + mountPath: /etc/localtime + - name: share + mountPath: /data/ascend-ci-share-pkking-sglang + - name: share + mountPath: /root/.cache + - name: share + mountPath: /root/sglang + subPath: {{ sglang_source_relative_path }} + volumes: + {% if env in ["ci", "debug"] %} + - name: share + persistentVolumeClaim: + claimName: sglang-guiyang004 + {% endif %} + {% if env == "green" %} + - name: share + hostPath: + path: /home + {% endif %} + - name: ascend-driver + hostPath: + path: /usr/local/Ascend/driver + - name: localtime + hostPath: + path: /etc/localtime + {% if env == "debug" %} + tolerations: + - key: "instance" + operator: "Equal" + value: "npu-class-service" + effect: "NoSchedule" + {% endif %} + nodeSelector: + {% if env in ["ci", "debug"] %} + accelerator/huawei-npu: ascend-snt9c + {% endif %} + {% if env == "debug" %} + node-status: debug + {% endif %} + {% if env == "green" %} + accelerator: huawei-Ascend910 + accelerator-type: module-a3-16 + {% endif %} + restartPolicy: OnFailure diff --git a/python/sglang/test/ascend/e2e/k8s_multi_pd_mix_green.yaml.jinja2 b/python/sglang/test/ascend/e2e/k8s_multi_pd_mix_green.yaml.jinja2 new file mode 100644 index 000000000..f1c971f98 --- /dev/null +++ b/python/sglang/test/ascend/e2e/k8s_multi_pd_mix_green.yaml.jinja2 @@ -0,0 +1,107 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ kube_config_map }} + namespace: {{ name_space }} +data: {} + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ kube_job_name }}-sglang-node + namespace: {{ name_space }} +spec: + replicas: {{ node_size }} + serviceName: {{ kube_job_name }}-sglang-pd-mix-headless + selector: + matchLabels: + app: sgl-ascend + task: pd-mix + template: + metadata: + labels: + app: sgl-ascend + task: pd-mix + spec: + hostNetwork: True + containers: + - image: {{ image }} + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + name: sgl-ascend + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: INSTALL_SGLANG_FROM_SOURCE + value: "{{ install_sglang_from_source }}" + - name: KUBECONFIG + value: "{{ kube_config }}" + - name: NAMESPACE + value: "{{ name_space }}" + - name: KUBE_CONFIG_MAP + value: "{{ kube_config_map }}" + - name: HF_ENDPOINT + value: "https://hf-mirror.com" + - name: METRICS_DATA_FILE + value: "{{ metrics_data_file }}" + - name: SGLANG_IS_IN_CI + value: "{{ sglang_is_in_ci }}" + - name: TRANSFORMERS_VERSION_FOR_SGLANG + value: "{{ transformers_version }}" + command: ["/bin/bash", "-c"] + args: + - | + bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }} + resources: + requests: + huawei.com/Ascend910: 16 + limits: + huawei.com/Ascend910: 16 + volumeMounts: + - name: ascend-driver + mountPath: /usr/local/Ascend/driver + - name: localtime + mountPath: /etc/localtime + - name: share + mountPath: /data/ascend-ci-share-pkking-sglang + - name: share + mountPath: /root/.cache + - name: share + mountPath: /root/sglang + subPath: {{ sglang_source_relative_path }} + volumes: + - name: share + hostPath: + path: /home + - name: ascend-driver + hostPath: + path: /usr/local/Ascend/driver + - name: localtime + hostPath: + path: /etc/localtime + nodeSelector: + accelerator: huawei-Ascend910 + accelerator-type: module-a3-16 + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ kube_job_name }}-sglang-pd-mix-headless + namespace: {{ name_space }} +spec: + clusterIP: None + selector: + app: sgl-ascend + task: pd-mix + ports: + - port: 80 + name: dummy-port diff --git a/python/sglang/test/ascend/e2e/k8s_multi_pd_separation.yaml.jinja2 b/python/sglang/test/ascend/e2e/k8s_multi_pd_separation.yaml.jinja2 new file mode 100644 index 000000000..bbffb69a6 --- /dev/null +++ b/python/sglang/test/ascend/e2e/k8s_multi_pd_separation.yaml.jinja2 @@ -0,0 +1,350 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ kube_config_map }} + namespace: {{ name_space }} +data: {} + +--- +apiVersion: batch.volcano.sh/v1alpha1 +kind: Job +metadata: + name: {{ kube_job_name }} + namespace: {{ name_space }} + labels: + ring-controller.atlas: ascend-1980 + fault-scheduling: "force" +spec: + minAvailable: {{ prefill_size + decode_size + router_size }} + schedulerName: volcano + policies: + - event: PodEvicted + action: RestartJob + queue: default + tasks: + - name: "sglang-prefill" + replicas: {{ prefill_size }} + template: + metadata: + labels: + app: sgl-ascend + ring-controller.atlas: ascend-1980 + spec: + hostNetwork: True + containers: + - image: {{ image }} + imagePullPolicy: Always + securityContext: + privileged: true + name: sgl-ascend + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: INSTALL_SGLANG_FROM_SOURCE + value: "{{ install_sglang_from_source }}" + - name: KUBECONFIG + value: "{{ kube_config }}" + - name: NAMESPACE + value: "{{ name_space }}" + - name: KUBE_CONFIG_MAP + value: "{{ kube_config_map }}" + - name: HF_ENDPOINT + value: "https://hf-mirror.com" + - name: METRICS_DATA_FILE + value: "{{ metrics_data_file }}" + - name: SGLANG_IS_IN_CI + value: "{{ sglang_is_in_ci }}" + - name: TRANSFORMERS_VERSION_FOR_SGLANG + value: "{{ transformers_version }}" + command: ["/bin/bash", "-c"] + args: + - | + {% if env in ["ci", "debug"] %} + bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }} + {% endif %} + {% if env == "green" %} + bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase_green.sh {{ test_case }} + {% endif %} + resources: + requests: + {% if env in ["ci", "debug"] %} + huawei.com/ascend-1980: 16 + {% endif %} + {% if env == "green" %} + huawei.com/Ascend910: 16 + {% endif %} + limits: + {% if env in ["ci", "debug"] %} + huawei.com/ascend-1980: 16 + {% endif %} + {% if env == "green" %} + huawei.com/Ascend910: 16 + {% endif %} + volumeMounts: + - name: ascend-driver + mountPath: /usr/local/Ascend/driver + - name: localtime + mountPath: /etc/localtime + - name: share + mountPath: /data/ascend-ci-share-pkking-sglang + - name: share + mountPath: /root/.cache + - name: share + mountPath: /root/sglang + subPath: {{ sglang_source_relative_path }} + volumes: + {% if env in ["ci", "debug"] %} + - name: share + persistentVolumeClaim: + claimName: sglang-guiyang004 + {% endif %} + {% if env == "green" %} + - name: share + hostPath: + path: /home + {% endif %} + - name: ascend-driver + hostPath: + path: /usr/local/Ascend/driver + - name: localtime + hostPath: + path: /etc/localtime + {% if env == "debug" %} + tolerations: + - key: "instance" + operator: "Equal" + value: "npu-class-service" + effect: "NoSchedule" + {% endif %} + nodeSelector: + {% if env in ["ci", "debug"] %} + accelerator/huawei-npu: ascend-snt9c + {% endif %} + {% if env == "debug" %} + node-status: debug + {% endif %} + {% if env == "green" %} + accelerator: huawei-Ascend910 + accelerator-type: module-a3-16 + {% endif %} + restartPolicy: OnFailure + - name: "sglang-decode" + replicas: {{ decode_size }} + template: + metadata: + labels: + app: sgl-ascend + ring-controller.atlas: ascend-1980 + spec: + hostNetwork: True + containers: + - image: {{ image }} + imagePullPolicy: Always + securityContext: + privileged: true + name: sgl-ascend + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: INSTALL_SGLANG_FROM_SOURCE + value: "{{ install_sglang_from_source }}" + - name: KUBECONFIG + value: "{{ kube_config }}" + - name: NAMESPACE + value: "{{ name_space }}" + - name: KUBE_CONFIG_MAP + value: "{{ kube_config_map }}" + - name: HF_ENDPOINT + value: "https://hf-mirror.com" + - name: METRICS_DATA_FILE + value: "{{ metrics_data_file }}" + - name: SGLANG_IS_IN_CI + value: "{{ sglang_is_in_ci }}" + - name: TRANSFORMERS_VERSION_FOR_SGLANG + value: "{{ transformers_version }}" + command: ["/bin/bash", "-c"] + args: + - | + {% if env in ["ci", "debug"] %} + bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }} + {% endif %} + {% if env == "green" %} + bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase_green.sh {{ test_case }} + {% endif %} + resources: + requests: + {% if env in ["ci", "debug"] %} + huawei.com/ascend-1980: 16 + {% endif %} + {% if env == "green" %} + huawei.com/Ascend910: 16 + {% endif %} + limits: + {% if env in ["ci", "debug"] %} + huawei.com/ascend-1980: 16 + {% endif %} + {% if env == "green" %} + huawei.com/Ascend910: 16 + {% endif %} + volumeMounts: + - name: ascend-driver + mountPath: /usr/local/Ascend/driver + - name: localtime + mountPath: /etc/localtime + - name: share + mountPath: /data/ascend-ci-share-pkking-sglang + - name: share + mountPath: /root/.cache + - name: share + mountPath: /root/sglang + subPath: {{ sglang_source_relative_path }} + volumes: + {% if env in ["ci", "debug"] %} + - name: share + persistentVolumeClaim: + claimName: sglang-guiyang004 + {% endif %} + {% if env == "green" %} + - name: share + hostPath: + path: /home + {% endif %} + - name: ascend-driver + hostPath: + path: /usr/local/Ascend/driver + - name: localtime + hostPath: + path: /etc/localtime + {% if env == "debug" %} + tolerations: + - key: "instance" + operator: "Equal" + value: "npu-class-service" + effect: "NoSchedule" + {% endif %} + nodeSelector: + {% if env in ["ci", "debug"] %} + accelerator/huawei-npu: ascend-snt9c + {% endif %} + {% if env == "debug" %} + node-status: debug + {% endif %} + {% if env == "green" %} + accelerator: huawei-Ascend910 + accelerator-type: module-a3-16 + {% endif %} + restartPolicy: OnFailure + - name: "sglang-router" + replicas: {{ router_size }} + template: + metadata: + labels: + app: sgl-ascend + ring-controller.atlas: ascend-1980 + spec: + hostNetwork: True + containers: + - image: {{ image }} + imagePullPolicy: Always + securityContext: + privileged: true + name: sgl-ascend + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: INSTALL_SGLANG_FROM_SOURCE + value: "{{ install_sglang_from_source }}" + - name: METRICS_DATA_FILE + value: "{{ metrics_data_file }}" + - name: KUBECONFIG + value: "{{ kube_config }}" + - name: NAMESPACE + value: "{{ name_space }}" + - name: KUBE_CONFIG_MAP + value: "{{ kube_config_map }}" + - name: HF_ENDPOINT + value: "https://hf-mirror.com" + - name: SGLANG_IS_IN_CI + value: "{{ sglang_is_in_ci }}" + - name: TRANSFORMERS_VERSION_FOR_SGLANG + value: "{{ transformers_version }}" + command: ["/bin/bash", "-c"] + args: + - | + {% if env in ["ci", "debug"] %} + bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }} + {% endif %} + {% if env == "green" %} + bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase_green.sh {{ test_case }} + {% endif %} + resources: + requests: + cpu: "4" + limits: + cpu: "4" + volumeMounts: + - name: ascend-driver + mountPath: /usr/local/Ascend/driver + - name: localtime + mountPath: /etc/localtime + - name: share + mountPath: /data/ascend-ci-share-pkking-sglang + - name: share + mountPath: /root/.cache + - name: share + mountPath: /root/sglang + subPath: {{ sglang_source_relative_path }} + volumes: + {% if env in ["ci", "debug"] %} + - name: share + persistentVolumeClaim: + claimName: sglang-guiyang004 + {% endif %} + {% if env == "green" %} + - name: share + hostPath: + path: /home + {% endif %} + - name: ascend-driver + hostPath: + path: /usr/local/Ascend/driver + - name: localtime + hostPath: + path: /etc/localtime + {% if env == "debug" %} + tolerations: + - key: "instance" + operator: "Equal" + value: "npu-class-service" + effect: "NoSchedule" + {% endif %} + nodeSelector: + {% if env in ["ci", "debug"] %} + accelerator/huawei-npu: ascend-snt9c + {% endif %} + {% if env == "debug" %} + node-status: debug + {% endif %} + {% if env == "green" %} + accelerator: huawei-Ascend910 + accelerator-type: module-a3-16 + {% endif %} + restartPolicy: OnFailure diff --git a/python/sglang/test/ascend/e2e/k8s_multi_pd_separation_green.yaml.jinja2 b/python/sglang/test/ascend/e2e/k8s_multi_pd_separation_green.yaml.jinja2 new file mode 100644 index 000000000..fa8f5b04e --- /dev/null +++ b/python/sglang/test/ascend/e2e/k8s_multi_pd_separation_green.yaml.jinja2 @@ -0,0 +1,307 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ kube_config_map }} + namespace: {{ name_space }} +data: {} + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ kube_job_name }}-sglang-prefill + namespace: {{ name_space }} +spec: + replicas: {{ prefill_size }} + serviceName: {{ kube_job_name }}-sglang-prefill-headless + selector: + matchLabels: + app: sgl-ascend + task: prefill + template: + metadata: + labels: + app: sgl-ascend + task: prefill + spec: + restartPolicy: Always + hostNetwork: True + containers: + - image: {{ image }} + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + name: sgl-ascend + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: INSTALL_SGLANG_FROM_SOURCE + value: "{{ install_sglang_from_source }}" + - name: KUBECONFIG + value: "{{ kube_config }}" + - name: NAMESPACE + value: "{{ name_space }}" + - name: KUBE_CONFIG_MAP + value: "{{ kube_config_map }}" + - name: HF_ENDPOINT + value: "https://hf-mirror.com" + - name: METRICS_DATA_FILE + value: "{{ metrics_data_file }}" + - name: SGLANG_IS_IN_CI + value: "{{ sglang_is_in_ci }}" + command: ["/bin/bash", "-c"] + args: + - | + bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }} + exit $? + resources: + requests: + huawei.com/Ascend910: 16 + limits: + huawei.com/Ascend910: 16 + volumeMounts: + - name: ascend-driver + mountPath: /usr/local/Ascend/driver + - name: localtime + mountPath: /etc/localtime + - name: share + mountPath: /data/ascend-ci-share-pkking-sglang + - name: share + mountPath: /root/.cache + - name: share + mountPath: /root/sglang + subPath: {{ sglang_source_relative_path }} + volumes: + - name: share + hostPath: + path: /home + - name: ascend-driver + hostPath: + path: /usr/local/Ascend/driver + - name: localtime + hostPath: + path: /etc/localtime + nodeSelector: + accelerator: huawei-Ascend910 + accelerator-type: module-a3-16 + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ kube_job_name }}-sglang-prefill-headless + namespace: {{ name_space }} +spec: + clusterIP: None + selector: + app: sgl-ascend + task: prefill + ports: + - port: 80 + name: dummy-port + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ kube_job_name }}-sglang-decode + namespace: {{ name_space }} +spec: + replicas: {{ decode_size }} + serviceName: {{ kube_job_name }}-sglang-decode-headless + selector: + matchLabels: + app: sgl-ascend + task: decode + template: + metadata: + labels: + app: sgl-ascend + task: decode + spec: + restartPolicy: Always + hostNetwork: True + containers: + - image: {{ image }} + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + name: sgl-ascend + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: INSTALL_SGLANG_FROM_SOURCE + value: "{{ install_sglang_from_source }}" + - name: KUBECONFIG + value: "{{ kube_config }}" + - name: NAMESPACE + value: "{{ name_space }}" + - name: KUBE_CONFIG_MAP + value: "{{ kube_config_map }}" + - name: HF_ENDPOINT + value: "https://hf-mirror.com" + - name: METRICS_DATA_FILE + value: "{{ metrics_data_file }}" + - name: SGLANG_IS_IN_CI + value: "{{ sglang_is_in_ci }}" + command: ["/bin/bash", "-c"] + args: + - | + bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }} + exit $? + resources: + requests: + huawei.com/Ascend910: 16 + limits: + huawei.com/Ascend910: 16 + volumeMounts: + - name: ascend-driver + mountPath: /usr/local/Ascend/driver + - name: localtime + mountPath: /etc/localtime + - name: share + mountPath: /data/ascend-ci-share-pkking-sglang + - name: share + mountPath: /root/.cache + - name: share + mountPath: /root/sglang + subPath: {{ sglang_source_relative_path }} + volumes: + - name: share + hostPath: + path: /home + - name: ascend-driver + hostPath: + path: /usr/local/Ascend/driver + - name: localtime + hostPath: + path: /etc/localtime + nodeSelector: + accelerator: huawei-Ascend910 + accelerator-type: module-a3-16 + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ kube_job_name }}-sglang-decode-headless + namespace: {{ name_space }} +spec: + clusterIP: None + selector: + app: sgl-ascend + task: decode + ports: + - port: 80 + name: dummy-port + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ kube_job_name }}-sglang-router + namespace: {{ name_space }} +spec: + replicas: {{ router_size }} + serviceName: {{ kube_job_name }}-sglang-router-headless + selector: + matchLabels: + app: sgl-ascend + task: router + template: + metadata: + labels: + app: sgl-ascend + task: router + spec: + hostNetwork: True + containers: + - image: {{ image }} + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + name: sgl-ascend + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: INSTALL_SGLANG_FROM_SOURCE + value: "{{ install_sglang_from_source }}" + - name: KUBECONFIG + value: "{{ kube_config }}" + - name: NAMESPACE + value: "{{ name_space }}" + - name: KUBE_CONFIG_MAP + value: "{{ kube_config_map }}" + - name: HF_ENDPOINT + value: "https://hf-mirror.com" + - name: METRICS_DATA_FILE + value: "{{ metrics_data_file }}" + - name: SGLANG_IS_IN_CI + value: "{{ sglang_is_in_ci }}" + command: ["/bin/bash", "-c"] + args: + - | + bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }} + resources: + requests: + cpu: 4 + limits: + cpu: 4 + volumeMounts: + - name: ascend-driver + mountPath: /usr/local/Ascend/driver + - name: localtime + mountPath: /etc/localtime + - name: share + mountPath: /data/ascend-ci-share-pkking-sglang + - name: share + mountPath: /root/.cache + - name: share + mountPath: /root/sglang + subPath: {{ sglang_source_relative_path }} + volumes: + - name: share + hostPath: + path: /home + - name: ascend-driver + hostPath: + path: /usr/local/Ascend/driver + - name: localtime + hostPath: + path: /etc/localtime + nodeSelector: + accelerator: huawei-Ascend910 + accelerator-type: module-a3-16 + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ kube_job_name }}-sglang-router-headless + namespace: {{ name_space }} +spec: + clusterIP: None + selector: + app: sgl-ascend + task: router + ports: + - port: 80 + name: dummy-port diff --git a/python/sglang/test/ascend/e2e/k8s_single.yaml.jinja2 b/python/sglang/test/ascend/e2e/k8s_single.yaml.jinja2 new file mode 100644 index 000000000..dff5a5591 --- /dev/null +++ b/python/sglang/test/ascend/e2e/k8s_single.yaml.jinja2 @@ -0,0 +1,134 @@ +apiVersion: batch.volcano.sh/v1alpha1 +kind: Job +metadata: + name: {{ kube_job_name }} + namespace: {{ name_space }} + labels: + ring-controller.atlas: ascend-1980 + fault-scheduling: "force" +spec: + minAvailable: 1 + schedulerName: volcano + policies: + - event: PodEvicted + action: RestartJob + {% if env == "green" %} + queue: default + {% endif %} + tasks: + - name: "pod" + replicas: 1 + template: + metadata: + labels: + app: sgl-ascend + ring-controller.atlas: ascend-1980 + spec: + containers: + - image: {{ image }} + {% if env == "green" %} + imagePullPolicy: IfNotPresent + {% else %} + imagePullPolicy: Always + {% endif %} + securityContext: + privileged: true + name: sgl-ascend + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: INSTALL_SGLANG_FROM_SOURCE + value: "{{ install_sglang_from_source }}" + - name: METRICS_DATA_FILE + value: "{{ metrics_data_file }}" + - name: KUBECONFIG + value: "{{ kube_config }}" + - name: HF_ENDPOINT + value: "https://hf-mirror.com" + - name: TROUBLE_SHOTTING + value: "{{ trouble_shotting }}" + - name: TRANSFORMERS_VERSION_FOR_SGLANG + value: "{{ transformers_version }}" + command: ["/bin/bash", "-c"] + args: + - | + bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }} + resources: + requests: + {% if env in ["ci", "debug"] %} + huawei.com/ascend-1980: {{ npu_size }} + {% endif %} + {% if env == "green" %} + huawei.com/Ascend910: {{ npu_size }} + memory: 128Gi + {% endif %} + cpu: "46" + limits: + {% if env in ["ci", "debug"] %} + huawei.com/ascend-1980: {{ npu_size }} + {% endif %} + {% if env == "green" %} + huawei.com/Ascend910: {{ npu_size }} + memory: 128Gi + {% endif %} + cpu: "46" + volumeMounts: + - name: ascend-driver + mountPath: /usr/local/Ascend/driver + - name: shm-volume + mountPath: /dev/shm + - name: localtime + mountPath: /etc/localtime + - name: share + mountPath: /data/ascend-ci-share-pkking-sglang + - name: share + mountPath: /root/.cache + - name: share + mountPath: /root/sglang + subPath: {{ sglang_source_relative_path }} + volumes: + {% if env in ["ci", "debug"] %} + - name: share + persistentVolumeClaim: + claimName: sglang-guiyang004 + {% endif %} + {% if env == "green" %} + - name: share + hostPath: + path: /home + {% endif %} + - name: ascend-driver + hostPath: + path: /usr/local/Ascend/driver + - name: shm-volume + emptyDir: + medium: Memory + sizeLimit: "16Gi" + - name: localtime + hostPath: + path: /etc/localtime + {% if env == "debug" %} + tolerations: + - key: "instance" + operator: "Equal" + value: "npu-class-service" + effect: "NoSchedule" + {% endif %} + nodeSelector: + {% if env in ["ci", "debug"] %} + accelerator/huawei-npu: ascend-snt9c + {% endif %} + {% if env == "debug" %} + node-status: debug + {% endif %} + {% if env == "green" %} + accelerator: huawei-Ascend910 + accelerator-type: module-a3-16 + {% endif %} + restartPolicy: OnFailure diff --git a/python/sglang/test/ascend/e2e/run_evalscope.sh b/python/sglang/test/ascend/e2e/run_evalscope.sh new file mode 100755 index 000000000..4b882b7fc --- /dev/null +++ b/python/sglang/test/ascend/e2e/run_evalscope.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e + +PYTHON_ENV_FOR_EVALSCOPE=test_env_evalscope +PIP_FOR_EVALSCOPE=${PYTHON_ENV_FOR_EVALSCOPE}/bin/pip +EVALSCOPE_SOURCE_PATH=/root/.cache/.cache/evalscope +pip_mirror_source="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" + +if [ -d "${PYTHON_ENV_FOR_EVALSCOPE}" ]; then + echo "Virtual env ${PYTHON_ENV_FOR_EVALSCOPE} already exists, skip installation." + exit 0 +fi + +echo "===== Install evalscope in virtual env - Begin =====" +python -m venv ${PYTHON_ENV_FOR_EVALSCOPE} + +if [ ! -d "${EVALSCOPE_SOURCE_PATH}" ]; then + echo "The evalscope source does not exist: ${EVALSCOPE_SOURCE_PATH}." + echo "Install evalscope online." + ${PIP_FOR_EVALSCOPE} install -U pip -i ${pip_mirror_source} + ${PIP_FOR_EVALSCOPE} install evalscope -i ${pip_mirror_source} +else + echo "Install evalscope from local source: ${EVALSCOPE_SOURCE_PATH}" + ${PIP_FOR_EVALSCOPE} install -U pip -i ${pip_mirror_source} + ${PIP_FOR_EVALSCOPE} install -e ${EVALSCOPE_SOURCE_PATH} -i ${pip_mirror_source} +fi +echo "===== Install evalscope in virtual env - End =====" diff --git a/python/sglang/test/ascend/e2e/run_npu_e2e_test.py b/python/sglang/test/ascend/e2e/run_npu_e2e_test.py new file mode 100644 index 000000000..4fff84480 --- /dev/null +++ b/python/sglang/test/ascend/e2e/run_npu_e2e_test.py @@ -0,0 +1,890 @@ +import argparse +import json +import logging +import os +import random +import re +import string +import subprocess +import time +import uuid + +import psutil +import yaml +from jinja2 import Template +from kubernetes import client, config +from kubernetes.client.rest import ApiException + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", + handlers=[logging.StreamHandler()], +) +logger = logging.getLogger(__name__) + +KUBE_CONFIG = os.environ.get("KUBECONFIG") +logger.info(f"KUBE_CONFIG: {KUBE_CONFIG}") +config.load_kube_config(KUBE_CONFIG) +core_api = client.CoreV1Api() +custom_api = client.CustomObjectsApi() +batch_api = client.BatchV1Api() +rbac_api = client.RbacAuthorizationV1Api() + +LOCAL_TIMEOUT = 10800 + +script_path = os.path.dirname(os.path.abspath(__file__)) + +KUBE_JOB_SINGLE = "single" +KUBE_JOB_MULTI_PD_MIX = "multi-pd-mix" +KUBE_JOB_MULTI_PD_SEPARATION = "multi-pd-separation" +KUBE_JOB_MULTI_PD_MIX_GREEN = "multi-pd-mix-green" +KUBE_JOB_MULTI_PD_SEPARATION_GREEN = "multi-pd-separation-green" +KUBE_YAML_TEMPLATE = { + KUBE_JOB_SINGLE: f"{script_path}/k8s_single.yaml.jinja2", + KUBE_JOB_MULTI_PD_MIX: f"{script_path}/k8s_multi_pd_mix.yaml.jinja2", + KUBE_JOB_MULTI_PD_MIX_GREEN: f"{script_path}/k8s_multi_pd_mix_green.yaml.jinja2", + KUBE_JOB_MULTI_PD_SEPARATION: f"{script_path}/k8s_multi_pd_separation.yaml.jinja2", + KUBE_JOB_MULTI_PD_SEPARATION_GREEN: f"{script_path}/k8s_multi_pd_separation_green.yaml.jinja2", +} + + +def get_unique_random_string(length: int = 16, add_random: bool = True) -> str: + """Generate a random string.""" + uuid_str = str(uuid.uuid4()).replace("-", "") + + if add_random: + if length < 8: + raise ValueError("length can not be smaller than 8") + random_length = length - 8 + char_pool = string.ascii_lowercase + string.digits + random_chars = "".join([random.choice(char_pool) for _ in range(random_length)]) + result = uuid_str[:8] + random_chars + else: + result = uuid_str[:length] + + return result + + +def create_kube_yaml(kube_yaml_template, output_yaml, pod_context): + """Create a k8s config yaml file""" + with open(kube_yaml_template, "r") as f: + template = Template(f.read()) + kube_pod_yaml = template.render(pod_context) + with open(output_yaml, "w") as f: + f.write(kube_pod_yaml) + logger.info(f"Pod YAML written to {output_yaml}") + + +def create_pod(yaml_file, namespace): + """Create a pod by k8s config yaml file""" + with open(yaml_file, "r", encoding="utf-8") as f: + yaml_docs = list(yaml.safe_load_all(f)) + + for doc in yaml_docs: + if not doc: + continue + + kind = doc.get("kind") + api_version = doc.get("apiVersion") + + try: + if kind == "Pod" and api_version == "v1": + core_api.create_namespaced_pod(namespace=namespace, body=doc) + logger.info(f"Pod {doc['metadata']['name']} created") + + elif kind == "Job" and api_version == "batch/v1": + batch_api.create_namespaced_job(namespace=namespace, body=doc) + logger.info(f"Job {doc['metadata']['name']} is created") + + elif kind == "Job" and api_version == "batch.volcano.sh/v1alpha1": + response = custom_api.create_namespaced_custom_object( + group="batch.volcano.sh", + version="v1alpha1", + namespace=namespace, + plural="jobs", + body=doc, + ) + logger.info(f"Volcano job {doc['metadata']['name']} is created") + logger.debug(response) + + elif kind == "ConfigMap" and api_version == "v1": + core_api.create_namespaced_config_map(namespace=namespace, body=doc) + logger.info(f"ConfigMap {doc['metadata']['name']} is created") + + elif kind == "Role" and api_version == "rbac.authorization.k8s.io/v1": + rbac_api.create_namespaced_role(namespace=namespace, body=doc) + logger.info(f"Role {doc['metadata']['name']} is created") + + elif ( + kind == "RoleBinding" and api_version == "rbac.authorization.k8s.io/v1" + ): + rbac_api.create_namespaced_role_binding(namespace=namespace, body=doc) + logger.info(f"RoleBinding {doc['metadata']['name']} is created") + + elif kind == "Deployment" and api_version == "apps/v1": + apps_api = client.AppsV1Api() + apps_api.create_namespaced_deployment(namespace=namespace, body=doc) + logger.info(f"Deployment {doc['metadata']['name']} is created") + + elif kind == "StatefulSet" and api_version == "apps/v1": + apps_api = client.AppsV1Api() + apps_api.create_namespaced_stateful_set(namespace=namespace, body=doc) + logger.info(f"StatefulSet {doc['metadata']['name']} is created") + + elif kind == "Service" and api_version == "v1": + core_api.create_namespaced_service(namespace=namespace, body=doc) + logger.info(f"Service {doc['metadata']['name']} is created") + + else: + raise f"Unrecognized kind: {kind}/{api_version}" + except ApiException as e: + print(f"create resource {kind} error: {e}") + raise + + +def delete_pod(yaml_file, namespace): + """Delete k8s pod by config yaml file""" + with open(yaml_file, "r", encoding="utf-8") as f: + yaml_docs = list(yaml.safe_load_all(f)) + for doc in yaml_docs: + if not doc: + continue + + kind = doc.get("kind") + api_version = doc.get("apiVersion") + try: + if kind == "Job" and api_version == "batch.volcano.sh/v1alpha1": + job_name = doc["metadata"]["name"] + response = custom_api.delete_namespaced_custom_object( + group="batch.volcano.sh", + version="v1alpha1", + namespace=namespace, + plural="jobs", + name=job_name, + body=client.V1DeleteOptions( + grace_period_seconds=0, propagation_policy="Foreground" + ), + ) + logger.info(f"Deleted job {job_name}") + logger.info(f"Response status: {response.get('status')}") + elif kind == "ConfigMap" and api_version == "v1": + config_map_name = doc["metadata"]["name"] + core_api.delete_namespaced_config_map( + name=config_map_name, namespace=namespace + ) + print(f"ConfigMap {config_map_name} is deleted.") + elif kind == "Deployment" and api_version == "apps/v1": + deployment_name = doc["metadata"]["name"] + apps_api = client.AppsV1Api() + apps_api.delete_namespaced_deployment( + name=deployment_name, + namespace=namespace, + body=client.V1DeleteOptions( + grace_period_seconds=0, propagation_policy="Foreground" + ), + ) + logger.info(f"Deployment {deployment_name} is deleted.") + + elif kind == "StatefulSet" and api_version == "apps/v1": + statefulset_name = doc["metadata"]["name"] + apps_api = client.AppsV1Api() + apps_api.delete_namespaced_stateful_set( + name=statefulset_name, + namespace=namespace, + body=client.V1DeleteOptions( + grace_period_seconds=0, propagation_policy="Foreground" + ), + ) + logger.info(f"StatefulSet {statefulset_name} is deleted.") + + elif kind == "Service" and api_version == "v1": + service_name = doc["metadata"]["name"] + core_api.delete_namespaced_service( + name=service_name, + namespace=namespace, + body=client.V1DeleteOptions( + grace_period_seconds=0, propagation_policy="Foreground" + ), + ) + logger.info(f"Service {service_name} is deleted.") + + else: + raise f"Unrecognized kind: {kind}/{api_version}" + except ApiException as e: + raise f"delete resource {kind} error: {e}" + + +def check_parent_process(): + """Check parent process is alive or not.""" + try: + parent_pid = os.getppid() + psutil.Process(parent_pid) + return True + except psutil.NoSuchProcess: + return False + + +def check_pods_ready(namespace, pod_name_key_str, timeout=300): + """Waiting for all k8s pods are ready""" + logger.info("Waiting all pods to running...") + start_time = time.time() + + while time.time() - start_time < timeout: + if not check_parent_process(): + raise Exception("Parent process exited.") + + pods = core_api.list_namespaced_pod(namespace=namespace) + + if len(pods.items) == 0: + time.sleep(5) + continue + + all_running = True + sglang_pods_found = False + for pod in pods.items: + pod_name = pod.metadata.name + if pod_name_key_str not in pod_name: + continue + + sglang_pods_found = True + status = pod.status + phase = status.phase + logger.info(f"Pod: {pod_name}, status: {phase}") + if phase != "Running": + all_running = False + break + + containers_ready = True + for condition in status.conditions: + if condition.type == "Ready" and condition.status != "True": + containers_ready = False + break + + if not containers_ready: + all_running = False + break + + if not sglang_pods_found: + logger.info("No sglang pod, waiting...") + time.sleep(5) + continue + if all_running: + logger.info("All sglang Pod is Running !") + return True + + time.sleep(5) + + logger.info(f"timeout in {timeout}s") + return False + + +def create_or_update_configmap(cm_name: str, data: dict, namespace: str): + """Create a k8s configmap or update it if already exists""" + cm_metadata = client.V1ObjectMeta(name=cm_name, namespace=namespace) + configmap = client.V1ConfigMap( + api_version="v1", kind="ConfigMap", metadata=cm_metadata, data=data + ) + + try: + response = core_api.create_namespaced_config_map( + namespace=namespace, body=configmap + ) + logger.info(f"ConfigMap '{cm_name}' create successfully!") + logger.info(f"data: {list(data.keys())}") + return response + except ApiException as e: + if e.status == 409: + logger.info(f"ConfigMap {cm_name} already exists. Updating...") + response = core_api.replace_namespaced_config_map( + namespace=namespace, name=cm_name, body=configmap + ) + logger.info(f"ConfigMap {cm_name} updated successfully.") + return response + else: + error_msg = f"ConfigMap create failed: {e.reason}" + if e.body: + error_msg += f" | details: {e.body}" + logger.info(error_msg) + raise + + +def prepare_cm_data(namespace, pod_string): + """Prepare a configmap data: {pod_name: pod_ip} by the running pod's information.""" + pods = core_api.list_namespaced_pod(namespace=namespace) + data = {} + for pod in pods.items: + pod_name = pod.metadata.name + if pod_string in pod_name: + pod_ip = pod.status.pod_ip + data[pod_name] = pod_ip + return data + + +def monitor_pod_logs( + kube_job_type, kube_job_prefix_name, namespace, timeout=LOCAL_TIMEOUT +): + """Monitor the logs of the specified pod until the special pattern is matched or reaches its timeout.""" + monitor_pod_name = { + KUBE_JOB_SINGLE: f"{kube_job_prefix_name}-pod-0", + KUBE_JOB_MULTI_PD_MIX: f"{kube_job_prefix_name}-sglang-node-0", + KUBE_JOB_MULTI_PD_SEPARATION: f"{kube_job_prefix_name}-sglang-router-0", + } + pod_name = monitor_pod_name.get(kube_job_type) + + # Build kubectl command + cmd = ["kubectl", "logs", "-f", "-n", namespace, pod_name] + + # Define multiline pattern to match + pattern_lines = [ + r"^-{70,}$", + r"^Ran \d+ tests? in [\d.]+s$", + r"^$", + r"^(OK|FAILED \(errors=\d+\))$", + ] + patterns = [re.compile(line_pattern) for line_pattern in pattern_lines] + pattern_ok = re.compile(r"^OK$") + + process = None + try: + # Start kubectl logs process + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + bufsize=1, + ) + + logger.info(f"Starting to monitor logs for Pod: {pod_name}") + match_state = 0 + is_success = False + + # Use two threads: one for reading logs, one for checking pod status + import threading + + # Shared variables + match_event = threading.Event() + pod_error_event = threading.Event() + + def read_logs(): + """Thread function to read logs continuously""" + nonlocal is_success, match_state + + while process.poll() is None and not match_event.is_set(): + line = process.stdout.readline() + if line: + line = line.rstrip("\n") + print(line) + # Check if current line matches expected pattern + if match_state < len(patterns) and patterns[match_state].match( + line + ): + match_state += 1 + if match_state == len(patterns): + if pattern_ok.match(line): + is_success = True + logger.info("Detected complete test completion pattern!") + match_event.set() + else: + match_state = 0 + if patterns[0].match(line): + match_state = 1 + + # Read remaining output after process exits + if not match_event.is_set(): + remaining_output, stderr_output = process.communicate() + if remaining_output: + print(remaining_output) + if stderr_output: + logger.error(f"kubectl command error: {stderr_output}") + pod_error_event.set() + + def check_pods_running(namespace, pod_name_key_str): + """check pods are running""" + pods = core_api.list_namespaced_pod(namespace=namespace) + if len(pods.items) == 0: + logger.warning(f"No pods found in the namespace {namespace}") + return False + + for pod in pods.items: + pod_name = pod.metadata.name + if pod_name_key_str not in pod_name: + continue + status = pod.status + phase = status.phase + if phase != "Running": + logger.error(f"Pod {pod_name} is not running, status: {phase}") + return False + + return True + + def check_pod_status(): + """Thread function to check pod status periodically""" + start_time = time.time() + while not match_event.is_set() and not pod_error_event.is_set(): + if time.time() - start_time > timeout: + pod_error_event.set() + break + + if not check_parent_process(): + logger.error(f"Parent process exited. Exiting...") + pod_error_event.set() + break + + if not check_pods_running( + namespace=namespace, pod_name_key_str=kube_job_prefix_name + ): + logger.error( + f"Some pods are not running properly. Please check the sglang logs on these pods. Exiting..." + ) + pod_error_event.set() + break + + # Sleep for a short time before next check + time.sleep(0.5) + + # Start threads + log_thread = threading.Thread(target=read_logs) + status_thread = threading.Thread(target=check_pod_status) + + log_thread.daemon = True + status_thread.daemon = True + + log_thread.start() + status_thread.start() + + # Wait for either match event or error event + start_time = time.time() + while not match_event.is_set() and not pod_error_event.is_set(): + if time.time() - start_time > timeout: + raise Exception( + f"Timeout exceeded, the thread is {timeout} seconds long." + ) + time.sleep(0.1) + + # Check if pattern was successfully matched + if not match_event.is_set(): + if process.poll() is not None: + remaining_output, stderr_output = process.communicate() + if remaining_output: + logger.info(remaining_output) + if stderr_output: + raise Exception(f"kubectl command error: {stderr_output}") + else: + raise Exception( + "Pod logs ended but target pattern was not detected" + ) + else: + raise Exception("Monitoring ended but target pattern was not detected") + elif not is_success: + raise Exception("The test result was FAILED!") + else: + logger.info("The test result was OK!") + finally: + if process and process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + + +def generate_metrics_json(metrics_data_file, test_case, status): + log_file = os.path.join(metrics_data_file, "test_output.log") + + metrics = {} + baselines = {} + + if os.path.exists(log_file): + with open(log_file, "r") as f: + for line in f: + m = re.match(r"\[METRIC\] (\S+)=(\S+)", line.strip()) + if m: + key = m.group(1) + value = m.group(2) + try: + value = float(value) + except ValueError: + pass + if key.endswith("_baseline"): + baselines[key[:-9]] = value + else: + metrics[key] = value + else: + logger.warning(f"Metrics log file not found: {log_file}") + + tc_name = test_case.rsplit("/", 1)[-1].rsplit(".", 1)[0] + + test_type = "unknown" + parts = metrics_data_file.split("/") + for i, part in enumerate(parts): + if part == "output" and i + 1 < len(parts): + test_type = parts[i + 1] + break + + output = { + "test_case": tc_name, + "test_type": test_type, + "status": status, + "metrics": metrics, + "baselines": baselines, + } + + output_path = os.path.join(metrics_data_file, "metrics.json") + with open(output_path, "w") as f: + json.dump(output, f, indent=2) + logger.info(f"Metrics JSON written to {output_path}") + + with open("/tmp/metrics.json", "w") as f: + json.dump(output, f, indent=2) + logger.info("Metrics JSON written to /tmp/metrics.json") + + +def run_npu_e2e_test_case( + docker_image_url: str, + kube_name_space: str, + kube_job_type: str, + kube_job_name_prefix: str, + resource_info: dict, + sglang_source_relative_path: str, + metrics_data_file: str, + test_case: str, + sglang_is_in_ci=False, + install_sglang_from_source=False, + env="debug", + trouble_shotting=False, + transformers_version="", +): + """The method for running a npu e2e test case. + Args: + docker_image_url (str): the url of docker image for creating k8s pods. + kube_name_space (str): the namespace of the k8s. + kube_job_name_prefix (str): the prefix of the k8s job name which will be set as the prefix of the pod name. + resource_info (dict): the number of k8s nodes used by the testcase. + for pd-separation as: {"prefill_size": 1, "decode_size": 1, "router_size": 1}; + for pd-mix as: {"node_size": 2; single: {"npu_size": 4} + sglang_source_relative_path (str): the relative path of the sglang source on shared-disk. + metrics_data_file (str): the output path of the metrics data file, only for performance testing. + test_case (str): the test case relative path in sglang source root path. like test/registered/... + sglang_is_in_ci (bool): whether running in CI environment. + install_sglang_from_source (bool): whether installing sglang from source or use docker image directly. + env (str): the environment to run the test on. Choose one in ["debug", "ci"] + """ + random_str = get_unique_random_string(16, True) + + kube_config_map = f"sglang-configmap-{random_str}" + final_kube_job_name = f"{kube_job_name_prefix}-{random_str}" + + kube_yaml_file_dict = { + KUBE_JOB_SINGLE: f"k8s_single_{random_str}.yaml", + KUBE_JOB_MULTI_PD_MIX: f"k8s_multi_pd_mix_{random_str}.yaml", + KUBE_JOB_MULTI_PD_SEPARATION: f"k8s_multi_pd_separation_{random_str}.yaml", + } + kube_yaml_file = kube_yaml_file_dict.get(kube_job_type) + + try: + logger.info( + f"Apply k8s yaml... KUBE_NAME_SPACE:{kube_name_space}, KUBE_CONFIG_MAP:{kube_config_map}, " + f"KUBE_JOB_TYPE:{kube_job_type}, KUBE_YAML_FILE:{kube_yaml_file}" + ) + + if kube_job_type == KUBE_JOB_SINGLE: + k8s_context = { + "image": docker_image_url, + "name_space": kube_name_space, + "kube_job_name": final_kube_job_name, + "kube_config": KUBE_CONFIG, + "npu_size": resource_info["npu_size"], + "sglang_source_relative_path": sglang_source_relative_path, + "metrics_data_file": metrics_data_file, + "test_case": test_case, + "sglang_is_in_ci": sglang_is_in_ci, + "install_sglang_from_source": install_sglang_from_source, + "env": env, + "trouble_shotting": trouble_shotting, + "transformers_version": transformers_version, + } + create_kube_yaml( + kube_yaml_template=KUBE_YAML_TEMPLATE.get(kube_job_type), + output_yaml=kube_yaml_file, + pod_context=k8s_context, + ) + elif kube_job_type == KUBE_JOB_MULTI_PD_MIX: + k8s_context = { + "image": docker_image_url, + "name_space": kube_name_space, + "kube_job_name": final_kube_job_name, + "kube_config": KUBE_CONFIG, + "kube_config_map": kube_config_map, + "node_size": resource_info["node_size"], + "sglang_source_relative_path": sglang_source_relative_path, + "metrics_data_file": metrics_data_file, + "test_case": test_case, + "sglang_is_in_ci": sglang_is_in_ci, + "install_sglang_from_source": install_sglang_from_source, + "env": env, + "trouble_shotting": trouble_shotting, + "transformers_version": transformers_version, + } + template_key = ( + KUBE_JOB_MULTI_PD_MIX_GREEN if env == "green" else kube_job_type + ) + create_kube_yaml( + kube_yaml_template=KUBE_YAML_TEMPLATE.get(template_key), + output_yaml=kube_yaml_file, + pod_context=k8s_context, + ) + elif kube_job_type == KUBE_JOB_MULTI_PD_SEPARATION: + k8s_context = { + "image": docker_image_url, + "name_space": kube_name_space, + "kube_job_name": final_kube_job_name, + "kube_config": KUBE_CONFIG, + "kube_config_map": kube_config_map, + "prefill_size": resource_info["prefill_size"], + "decode_size": resource_info["decode_size"], + "router_size": resource_info["router_size"], + "sglang_source_relative_path": sglang_source_relative_path, + "metrics_data_file": metrics_data_file, + "test_case": test_case, + "sglang_is_in_ci": sglang_is_in_ci, + "install_sglang_from_source": install_sglang_from_source, + "env": env, + "trouble_shotting": trouble_shotting, + "transformers_version": transformers_version, + } + template_key = ( + KUBE_JOB_MULTI_PD_SEPARATION_GREEN if env == "green" else kube_job_type + ) + create_kube_yaml( + kube_yaml_template=KUBE_YAML_TEMPLATE.get(template_key), + output_yaml=kube_yaml_file, + pod_context=k8s_context, + ) + else: + raise Exception(f"Unknown k8s job type: {kube_job_type}") + + create_pod(yaml_file=kube_yaml_file, namespace=kube_name_space) + + if check_pods_ready( + kube_name_space, final_kube_job_name, timeout=LOCAL_TIMEOUT + ): + if kube_job_type != "single": + matching_pod_string = final_kube_job_name + cm_data = prepare_cm_data(kube_name_space, matching_pod_string) + if not cm_data: + logger.info( + f"No sglang pod found while matching {matching_pod_string}" + ) + + response = create_or_update_configmap( + cm_name=kube_config_map, data=cm_data, namespace=kube_name_space + ) + logger.info(response) + else: + logger.info("Pod not ready, maybe not enough resource") + + monitor_success = False + try: + monitor_pod_logs( + kube_job_type, final_kube_job_name, kube_name_space, LOCAL_TIMEOUT + ) + monitor_success = True + except Exception: + logger.error(f"Test case failed: {test_case}", exc_info=True) + raise + finally: + if metrics_data_file: + status = "pass" if monitor_success else "fail" + try: + generate_metrics_json(metrics_data_file, test_case, status) + except Exception as e: + logger.error(f"Failed to generate metrics JSON: {e}", exc_info=True) + finally: + if os.path.exists(kube_yaml_file): + # Don't delete pod when trouble_shotting is enabled + if not trouble_shotting: + delete_pod(yaml_file=kube_yaml_file, namespace=kube_name_space) + os.remove(kube_yaml_file) + else: + logger.info( + f"Trouble shooting mode enabled, keeping pod {final_kube_job_name} alive" + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Apply k8s yaml", formatter_class=argparse.RawTextHelpFormatter + ) + + parser.add_argument( + "--image", + type=str, + required=True, + help="Docker image to use", + ) + + parser.add_argument( + "--prefill-size", + type=int, + required=False, + default=1, + help="Number of prefill nodes", + ) + + parser.add_argument( + "--decode-size", + type=int, + required=False, + default=1, + help="Number of decode nodes", + ) + + parser.add_argument( + "--router-size", + type=int, + required=False, + default=1, + help="Number of router nodes", + ) + + parser.add_argument( + "--node-size", + type=int, + required=False, + default=2, + help="Number of nodes for multi-node-pd-mix scenario", + ) + + parser.add_argument( + "--npu-size", + type=int, + required=False, + default=0, + help="Number of npu for single-node scenario", + ) + + parser.add_argument( + "--sglang-source-relative-path", + type=str, + required=True, + help="Sglang source code relative path on shared-disk(NFS_ROOT_PATH: /data/ascend-ci-share-pkking-sglang/)", + ) + + parser.add_argument( + "--metrics-data-file", + type=str, + required=False, + default="", + help="Metrics data file", + ) + + parser.add_argument( + "--test-case", + type=str, + required=True, + help="Test case path", + ) + + parser.add_argument( + "--sglang-is-in-ci", + action="store_true", + help="Used to set env var SGLANG_IS_IN_CI in pod", + ) + + parser.add_argument( + "--install-sglang-from-source", + action="store_true", + help="Used to set env var INSTALL_SGLANG_FROM_SOURCE in pod", + ) + + parser.add_argument( + "--kube-name-space", + type=str, + required=True, + help="K8s name space", + ) + + parser.add_argument( + "--kube-job-type", + type=str, + choices=[KUBE_JOB_SINGLE, KUBE_JOB_MULTI_PD_MIX, KUBE_JOB_MULTI_PD_SEPARATION], + required=True, + help=f"K8s job type [{KUBE_JOB_SINGLE}, {KUBE_JOB_MULTI_PD_MIX}, {KUBE_JOB_MULTI_PD_SEPARATION}]", + ) + + parser.add_argument( + "--kube-job-name-prefix", + type=str, + required=True, + help="K8s job name prefix", + ) + + parser.add_argument( + "--env", + type=str, + choices=["debug", "ci", "green"], + required=True, + help="Environment type", + ) + + parser.add_argument( + "--trouble-shotting", + action="store_true", + help="Used for troubleshotting issues, such as retaining pods", + ) + + parser.add_argument( + "--transformers-version", + type=str, + required=False, + default="", + help="The transformers version number for running sglang. Use default version in image if keep empty.", + ) + + args = parser.parse_args() + + docker_image_url = args.image + npu_size = int(args.npu_size) + node_size = int(args.node_size) + prefill_size = int(args.prefill_size) + decode_size = int(args.decode_size) + router_size = int(args.router_size) + sglang_source_relative_path = args.sglang_source_relative_path + metrics_data_file = args.metrics_data_file + test_case = args.test_case + sglang_is_in_ci = args.sglang_is_in_ci + install_sglang_from_source = args.install_sglang_from_source + env = args.env + trouble_shotting = args.trouble_shotting + transformers_version = args.transformers_version + + kube_name_space = args.kube_name_space + kube_job_type = args.kube_job_type + kube_job_name_prefix = args.kube_job_name_prefix + + resource_info_dict = { + KUBE_JOB_SINGLE: {"npu_size": npu_size}, + KUBE_JOB_MULTI_PD_MIX: {"node_size": node_size}, + KUBE_JOB_MULTI_PD_SEPARATION: { + "prefill_size": prefill_size, + "decode_size": decode_size, + "router_size": router_size, + }, + } + + run_npu_e2e_test_case( + docker_image_url=docker_image_url, + kube_name_space=kube_name_space, + kube_job_type=kube_job_type, + kube_job_name_prefix=kube_job_name_prefix, + resource_info=resource_info_dict.get(kube_job_type), + sglang_source_relative_path=sglang_source_relative_path, + metrics_data_file=metrics_data_file, + test_case=test_case, + sglang_is_in_ci=sglang_is_in_ci, + install_sglang_from_source=install_sglang_from_source, + env=env, + trouble_shotting=trouble_shotting, + transformers_version=transformers_version, + ) diff --git a/python/sglang/test/ascend/e2e/run_npu_testcase.sh b/python/sglang/test/ascend/e2e/run_npu_testcase.sh new file mode 100644 index 000000000..439fdb6a9 --- /dev/null +++ b/python/sglang/test/ascend/e2e/run_npu_testcase.sh @@ -0,0 +1,151 @@ +test_case=$1 + +sglang_source_path=/root/sglang +if [ ! -f "${sglang_source_path}/${test_case}" ];then + echo "The test case file is not exist: $test_case" + exit 0 +fi + +echo "NPU info:" +npu-smi info + +echo "===== Install kubernetes - Begin =====" +KUBERNETES_PKG_PATH_SOURCE=/root/.cache/.cache/kubernetes +if [ ! -d "${KUBERNETES_PKG_PATH_SOURCE}" ]; then + echo "Install kubernetes online." + pip install kubernetes -i -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple +else + echo "Install kubernetes locally." + cp -r ${KUBERNETES_PKG_PATH_SOURCE} /tmp/ + pip install --no-index --find-links=/tmp/kubernetes/ kubernetes +fi +echo "===== Install kubernetes - End =====" + +PYTHON_FOR_SGLANG="python" +PIP_FOR_SGLANG="pip" +if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then + echo "===== Install transformers for sglang - Begin =====" + TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG} + if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then + echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}." + echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} online." + pip install transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple + else + echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally." + TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG} + mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}" + cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/" + pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" + fi + echo "===== Install transformers for sglang in virtual env - End =====" +fi + +if [ -n "${TRANSFORMERS_VERSION_FOR_TEST_TOOL}" ]; then + # Example: TRANSFORMERS_VERSION_FOR_TEST_TOOL=4.57.6 + echo "===== Install transformers in virtual env for test tools - Begin =====" + PYTHON_ENV_FOR_TEST_TOOL=python_venv_for_test_tool + PIP_FOR_TEST_TOOL=${PYTHON_ENV_FOR_TEST_TOOL}/bin/pip + python -m venv ${PYTHON_ENV_FOR_TEST_TOOL} --system-site-packages + TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_TEST_TOOL} + if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then + echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}." + echo "Install transformers ${TRANSFORMERS_VERSION_FOR_TEST_TOOL} online." + ${PIP_FOR_TEST_TOOL} install transformers==${TRANSFORMERS_VERSION_FOR_TEST_TOOL} -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple + else + echo "Install transformers ${TRANSFORMERS_VERSION_FOR_TEST_TOOL} locally." + TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_TEST_TOOL} + mkdir -p ${TRANSFORMERS_PKG_PATH_TARGET} + cp ${TRANSFORMERS_PKG_PATH_SOURCE}/* ${TRANSFORMERS_PKG_PATH_TARGET}/ + ${PIP_FOR_TEST_TOOL} install --no-index --find-links=${TRANSFORMERS_PKG_PATH_TARGET} transformers==${TRANSFORMERS_VERSION_FOR_TEST_TOOL} + fi + echo "===== Install transformers in virtual env for test tools - End =====" + echo "Transformers version for test tools: $(${PIP_FOR_TEST_TOOL} show transformers | grep Version | cut -d: -f2)" +fi + +echo "Transformers version for sglang: $(${PIP_FOR_SGLANG} show transformers | grep Version | cut -d: -f2)" + +# copy or download required file +cp /root/.cache/huggingface/hub/datasets--anon8231489123--ShareGPT_Vicuna_unfiltered/snapshots/192ab2185289094fc556ec8ce5ce1e8e587154ca/ShareGPT_V3_unfiltered_cleaned_split.json /tmp +#curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl +cp /root/.cache/modelscope/hub/datasets/grade_school_math/test.jsonl /tmp + +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +export SGLANG_TEST_MAX_RETRY=0 +export SGLANG_SET_CPU_AFFINITY=1 +export HCCL_HOST_SOCKET_PORT_RANGE="auto" +export HCCL_NPU_SOCKET_PORT_RANGE="auto" + +visibe_devices=$ASCEND_VISIBLE_DEVICES +echo "ASCEND_VISIBLE_DEVICES=$ASCEND_VISIBLE_DEVICES" +if [ "${visibe_devices}" != "" ];then + ASCEND_RT_VISIBLE_DEVICES=$(echo "$ASCEND_VISIBLE_DEVICES" | tr ',' '\n' | sort -n | tr '\n' ',') + export ASCEND_RT_VISIBLE_DEVICES=${ASCEND_RT_VISIBLE_DEVICES%,} + echo "ASCEND_RT_VISIBLE_DEVICES=$ASCEND_RT_VISIBLE_DEVICES" + export ASCEND_VISIBLE_DEVICES="" +fi + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +# use sglang from source or from image +if [ "${INSTALL_SGLANG_FROM_SOURCE}" = "true" ] || [ "${INSTALL_SGLANG_FROM_SOURCE}" = "True" ];then + echo "Use sglang from source: ${sglang_source_path}" + export PYTHONPATH=${sglang_source_path}/python:$PYTHONPATH +else + echo "Use sglang from docker image" + sglang_pkg_path=/sgl-workspace/sglang/python + ascend_test_util_path=${sglang_pkg_path}/sglang/test/ascend + mkdir -p "${ascend_test_util_path}" + mv "${ascend_test_util_path}" "${ascend_test_util_path}_bak" + cp -r ${sglang_source_path}/python/sglang/test/ascend "${ascend_test_util_path}" +fi + +# set environment of cann +. /usr/local/Ascend/cann/set_env.sh +. /usr/local/Ascend/nnal/atb/set_env.sh + +echo "Running test case ${test_case}" +tc_name=${test_case##*/} +tc_name=${tc_name%.*} +current_date=$(date +%Y%m%d) +log_path="/root/sglang/debug/logs/log/${current_date}/${tc_name}/${HOSTNAME}" +if [ "${SGLANG_IS_IN_CI}" = "true" ] || [ "${SGLANG_IS_IN_CI}" = "True" ];then + log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}" +fi +rm -rf "${log_path}" +mkdir -p "${log_path}" +echo "Log path: ${log_path}" + +if [ "${TROUBLE_SHOTTING}" = "true" ] || [ "${TROUBLE_SHOTTING}" = "True" ];then + echo "TROUBLE_SHOTTING=true, the pod will keep alive for four hour." + ( ${PYTHON_FOR_SGLANG} -u "${sglang_source_path}/${test_case}" 2>&1 || true ) | tee -a "${log_path}/${tc_name}.log" + sleep 14400 +else + ${PYTHON_FOR_SGLANG} -u "${sglang_source_path}/${test_case}" 2>&1 | tee -a "${log_path}/${tc_name}.log" +fi +echo "Finished test case ${test_case}" + +if [ -n "${METRICS_DATA_FILE}" ]; then + mkdir -p "${METRICS_DATA_FILE}" + cp "${log_path}/${tc_name}.log" "${METRICS_DATA_FILE}/test_output.log" + echo "Metrics log saved to ${METRICS_DATA_FILE}/test_output.log" +fi + +source_plog_path="/root/ascend/log/debug/plog" +if [ -d "$source_plog_path" ];then + echo "Plog files found. Begin to backup them." + target_plog_path="/root/sglang/debug/logs/plog/${tc_name}/${HOSTNAME}" + if [ "${SGLANG_IS_IN_CI}" = "true" ] || [ "${SGLANG_IS_IN_CI}" = "True" ];then + target_plog_path="/root/.cache/tests/logs/plog/${tc_name}/${HOSTNAME}" + fi + rm -rf "${target_plog_path}" + mkdir -p "${target_plog_path}" + cp ${source_plog_path}/* "${target_plog_path}" +fi diff --git a/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py b/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py new file mode 100644 index 000000000..58a63df18 --- /dev/null +++ b/python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py @@ -0,0 +1,525 @@ +import json +import logging +import os +import re +import subprocess +import threading +import time +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.e2e.test_npu_multi_node_utils import ( + SERVICE_PORT, + check_role, + launch_pd_mix_node, + launch_pd_separation_node, + launch_router, + wait_server_ready, +) +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + dump_metric, + popen_launch_server, +) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", + handlers=[logging.StreamHandler()], +) +logger = logging.getLogger(__name__) + +EVALSCOPE = "evalscope" +BENCHMARK_TOOL_DEFAULT = EVALSCOPE + +PYTHON_FOR_TEST_TOOL = "test_env_transformers_tool/bin/python" +if not os.path.exists(PYTHON_FOR_TEST_TOOL) or not os.access( + PYTHON_FOR_TEST_TOOL, os.X_OK +): + PYTHON_FOR_TEST_TOOL = "python3" +logger.info(f"PYTHON_FOR_TEST_TOOL: {PYTHON_FOR_TEST_TOOL}") + +DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 3600 +MAX_SERVER_KEEP_ALIVE_TIME = 3600 + +ACCURACY_TOLERANCE = 0.99 + +SERVER_INITIALIZATION_DELAY = 120 + +if os.environ.get("ASCEND_RT_VISIBLE_DEVICES"): + DEFAULT_SERVER_PORT_FOR_TEST = ( + 20000 + int(os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "0")[0]) * 100 + ) +else: + DEFAULT_SERVER_PORT_FOR_TEST = ( + 20000 + int(os.environ.get("ASCEND_VISIBLE_DEVICES", "0")[0]) * 100 + ) +DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_SERVER_PORT_FOR_TEST + 66}" + + +def run_evalscope( + host, + port, + model, + datasets, + dataset_args=None, + eval_batch_size=16, + limit=100000, + generation_config=None, + dataset_dir=None, + timeout=60000, + stream=True, + eval_type="openai_api", +): + + metrics_path = os.getenv("METRICS_DATA_FILE") + result_path = "./evalscope_result" if not metrics_path else metrics_path + logger.info(f"The metrics result file: {result_path}") + + api_url = f"http://{host}:{port}/v1/chat/completions" + + if generation_config is None: + generation_config = {"max_tokens": 512} + + config_dict = { + "model": model, + "api_url": api_url, + "eval_type": eval_type, + "datasets": datasets, + "eval_batch_size": eval_batch_size, + "generation_config": generation_config, + "timeout": timeout, + "stream": stream, + "limit": limit, + "work_dir": result_path, + } + if dataset_args: + config_dict["dataset_args"] = dataset_args + if dataset_dir: + config_dict["dataset_dir"] = dataset_dir + + config_json = json.dumps(config_dict, ensure_ascii=False, indent=2) + config_json_escaped = config_json.replace("\\", "\\\\").replace("'''", "\\'\\'\\'") + + script_content = "import json\n" + script_content += "from evalscope import TaskConfig, run_task\n\n" + script_content += f"config = json.loads('''{config_json_escaped}''')\n" + script_content += "task_cfg = TaskConfig(**config)\n" + script_content += "run_task(task_cfg=task_cfg)\n" + + script_path = f"/tmp/evalscope_run_{model}_{'_'.join(datasets)}.py" + with open(script_path, "w") as f: + f.write(script_content) + + logger.info(f"Generated evalscope script: {script_path}") + + install_cmd = ( + "/bin/bash /root/sglang/python/sglang/test/ascend/e2e/run_evalscope.sh" + ) + subprocess.run(install_cmd, shell=True, check=True) + + python_bin = "test_env_evalscope/bin/python" + cmd = f"{python_bin} {script_path}" + + logger.info(f"Command: {cmd}") + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + shell=True, + ) + + output_lines = [] + try: + for line in iter(process.stdout.readline, ""): + if line.strip(): + print(line, end="") + output_lines.append(line.strip()) + + process.wait() + + if process.returncode != 0: + logger.error(f"Command failed with return code: {process.returncode}") + raise subprocess.CalledProcessError(process.returncode, cmd) + + logger.info("Command executed successfully") + + metrics = {} + full_output = "\n".join(output_lines) + + report_match = re.search(r"Dump report to:\s*(\S+)", full_output) + if report_match: + report_path = report_match.group(1) + logger.info(f"Found evalscope report file: {report_path}") + try: + with open(report_path, "r") as rf: + report_data = json.load(rf) + for item in report_data: + score = item.get("score") + if score is not None: + metrics["accuracy"] = float(score) + logger.info(f"The Final Accuracy from report: {score}") + break + except Exception as e: + logger.warning(f"Failed to read report file {report_path}: {e}") + + if "accuracy" not in metrics: + accuracy_patterns = [ + r"mean_acc\s*.*?│\s*\d+\s*│\s*([\d.]+)\s*│", + r"│\s+([\d.]+)\s+│\s+\S+\s+│\s*$", + r"accuracy\s*[:=]?\s*([\d.]+)", + r"Accuracy\s*[:=]?\s*([\d.]+)", + r"score\s*[:=]?\s*([\d.]+)", + ] + + for pattern in accuracy_patterns: + matches = re.findall(pattern, full_output) + if matches: + final_accuracy = float(matches[-1]) + metrics["accuracy"] = final_accuracy + logger.info(f"The Final Accuracy from output: {final_accuracy}") + break + + if "accuracy" not in metrics: + logger.info("Can Not Find The Accuracy in evalscope output") + + return metrics + + except KeyboardInterrupt: + logger.info("Keyboard interrupt received, terminating process...") + process.terminate() + try: + process.wait(timeout=5) + logger.info("Process terminated") + except subprocess.TimeoutExpired: + logger.warning("Process did not terminate gracefully, killing it...") + process.kill() + logger.info("Process killed") + raise + except Exception as e: + logger.error(f"Error executing command: {e}") + process.terminate() + process.wait(timeout=5) + raise + + +def assert_metrics(self, metrics): + if not metrics: + raise Exception("No metrics obtained from benchmark") + + if self.accuracy is not None: + dump_metric( + "accuracy", + float(metrics["accuracy"]), + labels={"test_case": self.__class__.__name__, "type": "accuracy"}, + ) + dump_metric( + "accuracy_baseline", + float(self.accuracy), + labels={"test_case": self.__class__.__name__, "type": "accuracy"}, + ) + self.assertGreaterEqual( + float(metrics["accuracy"]), + self.accuracy * ACCURACY_TOLERANCE, + f"Accuracy check failed. Expected >= {self.accuracy * ACCURACY_TOLERANCE}, Got: {metrics['accuracy']}", + ) + + +MMMU_LOCAL_PATH = "/root/.cache/modelscope/hub/datasets/AI-ModelScope___mmmu" + + +class TestNpuAccuracyTestCaseBase(CustomTestCase): + model = None + benchmark_tool = BENCHMARK_TOOL_DEFAULT + backend = "sglang" + datasets = ["gsm8k"] + dataset_args = None + eval_batch_size = 16 + limit = 100000 + generation_config = None + dataset_dir = None + stream = True + timeout = 60000 + eval_type = "openai_api" + other_args = None + server_timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + envs = None + max_attempts = 2 + accuracy = 0.1 + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + env = os.environ.copy() + for key, value in env.items(): + logger.info(f"ENV_VAR_SYS {key}:{value}") + if cls.envs: + for key, value in cls.envs.items(): + logger.info(f"ENV_VAR_CASE {key}:{value}") + env[key] = value + + other_args = list(cls.other_args) + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=cls.server_timeout, + other_args=other_args, + env=env, + ) + + @classmethod + def tearDownClass(cls): + if hasattr(cls, "process") and cls.process: + try: + kill_process_tree(cls.process.pid) + except Exception as e: + logger.error(f"Error during tearDown: {e}") + + def _get_dataset_args(self): + if "mmmu" in self.datasets: + base_args = {"mmmu": {"dataset_id": MMMU_LOCAL_PATH}} + if self.dataset_args: + if isinstance(self.dataset_args, dict): + base_args.update(self.dataset_args) + elif isinstance(self.dataset_args, str): + base_args.update(json.loads(self.dataset_args)) + return base_args + return self.dataset_args + + def run_accuracy(self): + parsed_url = urlparse(self.base_url) + host = parsed_url.hostname + port = parsed_url.port + if self.benchmark_tool == EVALSCOPE: + model_name = os.path.basename(self.model) + metrics = run_evalscope( + host=host, + port=port, + model=model_name, + datasets=self.datasets, + dataset_args=self._get_dataset_args(), + eval_batch_size=self.eval_batch_size, + limit=self.limit, + generation_config=self.generation_config, + dataset_dir=self.dataset_dir, + stream=self.stream, + timeout=self.timeout, + eval_type=self.eval_type, + ) + assert_metrics(self, metrics) + + +class TestNpuAccuracyMultiNodePdMixTestCaseBase(CustomTestCase): + model_config = None + benchmark_tool = BENCHMARK_TOOL_DEFAULT + backend = "sglang" + datasets = ["gsm8k"] + dataset_args = None + eval_batch_size = 16 + limit = 100000 + generation_config = None + dataset_dir = None + stream = True + timeout = 60000 + eval_type = "openai_api" + other_args = None + server_timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + envs = None + max_attempts = 2 + accuracy = 0.1 + + @classmethod + def setUpClass(cls): + cls.local_ip = "127.0.0.1" + cls.host = os.getenv("POD_IP") + cls.port = SERVICE_PORT + cls.base_url = f"http://{cls.host}:{cls.port}" + cls.hostname = os.getenv("HOSTNAME") + cls.role = "master" if cls.hostname.endswith("sglang-node-0") else "worker" + logger.info(f"Init {cls.host} {cls.role=}!") + + cls.start_pd_mix_master_node() + cls.start_pd_mix_worker_node() + + @classmethod + def tearDownClass(cls): + pass + + @classmethod + @check_role(allowed_roles=["master"]) + def start_pd_mix_master_node(cls): + sglang_thread = threading.Thread( + target=launch_pd_mix_node, args=(cls.model_config,) + ) + sglang_thread.start() + + wait_server_ready(f"{cls.base_url}/health") + + logger.info( + f"Wait {SERVER_INITIALIZATION_DELAY}s, starting run benchmark ......" + ) + time.sleep(SERVER_INITIALIZATION_DELAY) + + @classmethod + @check_role(allowed_roles=["worker"]) + def start_pd_mix_worker_node(cls): + sglang_thread = threading.Thread( + target=launch_pd_mix_node, args=(cls.model_config,) + ) + sglang_thread.start() + + logger.info( + f"{cls.role} node started, keeping test alive for {MAX_SERVER_KEEP_ALIVE_TIME} seconds" + ) + time.sleep(MAX_SERVER_KEEP_ALIVE_TIME) + + def _get_dataset_args(self): + if "mmmu" in self.datasets: + base_args = {"mmmu": {"dataset_id": MMMU_LOCAL_PATH}} + if self.dataset_args: + if isinstance(self.dataset_args, dict): + base_args.update(self.dataset_args) + elif isinstance(self.dataset_args, str): + base_args.update(json.loads(self.dataset_args)) + return base_args + return self.dataset_args + + @check_role(allowed_roles=["master", "worker"]) + def run_accuracy(self): + parsed_url = urlparse(self.base_url) + host = parsed_url.hostname + port = parsed_url.port + if self.benchmark_tool == EVALSCOPE: + model_name = os.path.basename(self.model_config.get("model_path")) + metrics = run_evalscope( + host=self.host, + port=self.port, + model=model_name, + datasets=self.datasets, + dataset_args=self._get_dataset_args(), + eval_batch_size=self.eval_batch_size, + limit=self.limit, + generation_config=self.generation_config, + dataset_dir=self.dataset_dir, + stream=self.stream, + timeout=self.timeout, + eval_type=self.eval_type, + ) + assert_metrics(self, metrics) + + +class TestNpuAccuracyMultiNodePdSepTestCaseBase(CustomTestCase): + model_config = None + benchmark_tool = BENCHMARK_TOOL_DEFAULT + backend = "sglang" + datasets = ["gsm8k"] + dataset_args = None + eval_batch_size = 16 + limit = 100000 + generation_config = None + dataset_dir = None + stream = True + timeout = 60000 + eval_type = "openai_api" + other_args = None + server_timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + max_attempts = 2 + accuracy = 0.1 + + @classmethod + def setUpClass(cls): + cls.process = None + cls.local_ip = "127.0.0.1" + cls.host = os.getenv("POD_IP") + cls.port = SERVICE_PORT + cls.base_url = f"http://{cls.host}:{cls.port}" + cls.hostname = os.getenv("HOSTNAME") + cls.role = ( + "router" + if "router" in cls.hostname + else "prefill" if "prefill" in cls.hostname else "decode" + ) + logger.info(f"Init {cls.host} {cls.role=}!") + + cls.start_pd_server() + cls.start_router_server() + + @classmethod + def tearDownClass(cls): + if cls.process: + try: + kill_process_tree(cls.process.pid) + except Exception as e: + logger.error(f"Error during tearDown: {e}") + + @classmethod + @check_role(allowed_roles=["router"]) + def start_router_server(cls): + logger.info(f"Starting router in thread...") + sglang_thread = threading.Thread(target=launch_router, args=(cls.model_config,)) + sglang_thread.daemon = True + sglang_thread.start() + + health_check_url = f"{cls.base_url}/health" + logger.info(f"Waiting for router to be ready at {health_check_url}") + wait_server_ready(health_check_url) + + logger.info( + f"Waiting {SERVER_INITIALIZATION_DELAY} seconds for the server to fully initialize..." + ) + time.sleep(SERVER_INITIALIZATION_DELAY) + + @classmethod + @check_role(allowed_roles=["prefill", "decode"]) + def start_pd_server(cls): + logger.info(f"Starting pd separation node...") + cls.process = launch_pd_separation_node(cls.model_config) + logger.info(f"Pd separation node started with PID: {cls.process.pid}") + + while True: + if cls.process.poll() is None: + time.sleep(30) + else: + exit_code = cls.process.poll() + raise Exception( + f"Sglang process exited on node {cls.host} {cls.hostname} with exit code: {exit_code}" + ) + + def _get_dataset_args(self): + if "mmmu" in self.datasets: + base_args = {"mmmu": {"dataset_id": MMMU_LOCAL_PATH}} + if self.dataset_args: + if isinstance(self.dataset_args, dict): + base_args.update(self.dataset_args) + elif isinstance(self.dataset_args, str): + base_args.update(json.loads(self.dataset_args)) + return base_args + return self.dataset_args + + @check_role(allowed_roles=["router"]) + def run_accuracy(self): + parsed_url = urlparse(self.base_url) + host = parsed_url.hostname + port = parsed_url.port + if self.benchmark_tool == EVALSCOPE: + model_name = os.path.basename(self.model_config.get("model_path")) + metrics = run_evalscope( + host=host, + port=port, + model=model_name, + datasets=self.datasets, + dataset_args=self._get_dataset_args(), + eval_batch_size=self.eval_batch_size, + limit=self.limit, + generation_config=self.generation_config, + dataset_dir=self.dataset_dir, + stream=self.stream, + timeout=self.timeout, + eval_type=self.eval_type, + ) + assert_metrics(self, metrics) diff --git a/python/sglang/test/ascend/e2e/test_npu_multi_node_utils.py b/python/sglang/test/ascend/e2e/test_npu_multi_node_utils.py new file mode 100644 index 000000000..c652ed386 --- /dev/null +++ b/python/sglang/test/ascend/e2e/test_npu_multi_node_utils.py @@ -0,0 +1,1003 @@ +import logging +import os +import re +import socket +import subprocess +import threading +import time +from functools import wraps +from types import SimpleNamespace +from typing import Iterable, Union + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_gsm8k +from sglang.test.test_utils import CustomTestCase, popen_launch_server + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", + handlers=[logging.StreamHandler()], +) +logger = logging.getLogger(__name__) + +NAMESPACE = os.environ.get("NAMESPACE") +CONFIGMAP_NAME = os.environ.get("KUBE_CONFIG_MAP") + +LOCAL_TIMEOUT = 3600 +ALL_ROLE_SET = {"prefill", "decode", "router", "master", "worker"} + +# Port numbers +ASCEND_RT_VISIBLE_DEVICES = os.environ.get("ASCEND_RT_VISIBLE_DEVICES") +SERVICE_PORT = ( + 6677 + if not ASCEND_RT_VISIBLE_DEVICES + else 6677 + int(ASCEND_RT_VISIBLE_DEVICES.strip().split(",")[0]) +) +PREFILL_DECODE_PORT = 8000 +BOOTSTRAP_INIT_PORT = 8995 + +# Timeouts and delays +ROUTER_CONFIGMAP_TIMEOUT = 300 +SERVER_INITIALIZATION_DELAY = 30 + + +def get_nic_name(): + """ + Automatically identify the optimal network interface for SGLang multi-machine deployment + Returns: str - Valid network interface name; None - No valid interface found + """ + # Define virtual interface prefixes to exclude (k8s/docker common) + exclude_prefixes = [ + "lo", + "docker", + "tunl", + "cali", + "veth", + "br-", + "virbr", + "eth0@if", + "kube-", + "flannel", + "weave", + "cilium", + ] + + proc_net_dev = "/proc/net/dev" + if not os.path.exists(proc_net_dev): + logger.error("Error: /proc/net/dev not found (not a Linux system)") + return None + + # Store interfaces with traffic (rx_bytes + tx_bytes > 0) + interfaces_with_traffic = {} + + with open(proc_net_dev, "r") as f: + # Skip header lines (first 2 lines) + lines = f.readlines()[2:] + for line in lines: + line = line.strip() + if not line: + continue + + # Split interface name and stats (format: "ifname: rx_bytes rx_packets ... tx_bytes ...") + parts = re.split(r"\s+", line) + if len(parts) < 10: # Ensure enough stats fields + continue + + ifname = parts[0].rstrip(":") + + # Skip virtual interfaces + if any(ifname.startswith(prefix) for prefix in exclude_prefixes): + continue + + # Get rx/tx bytes (2nd field: rx_bytes, 10th field: tx_bytes) + try: + rx_bytes = int(parts[1]) + tx_bytes = int(parts[9]) + total_bytes = rx_bytes + tx_bytes + except (ValueError, IndexError): + continue + + # Only keep interfaces with traffic (active link) + if total_bytes > 0: + interfaces_with_traffic[ifname] = total_bytes + + # Priority 1: Select interface with most traffic (most active) + if interfaces_with_traffic: + # Sort by total bytes (descending) and pick first + sorted_interfaces = sorted( + interfaces_with_traffic.items(), key=lambda x: x[1], reverse=True + ) + nic_name = sorted_interfaces[0][0] + logger.info(f"The nic name matched is {nic_name}") + return nic_name + + # Priority 2: Fallback to first non-virtual interface (no traffic but exists) + # Re-read to get non-virtual interfaces (even with no traffic) + all_non_virtual = [] + with open(proc_net_dev, "r") as f: + lines = f.readlines()[2:] + for line in lines: + line = line.strip() + if not line: + continue + ifname = re.split(r"\s+", line)[0].rstrip(":") + if not any(ifname.startswith(p) for p in exclude_prefixes): + all_non_virtual.append(ifname) + + if all_non_virtual: + nic_name = all_non_virtual[0] + logger.info(f"The nic name matched is {nic_name}") + return nic_name + + # No valid interface found + logger.error("No valid interface found") + return None + + +nic = get_nic_name() +NIC_NAME = "lo" if nic is None else nic + + +def get_host_name(): + host_name = os.getenv("HOSTNAME") + if not host_name: + raise RuntimeError( + f"Missing required environment variables: HOSTNAME={host_name}" + ) + return host_name + + +def get_host_ip(): + host_ip = os.getenv("POD_IP") + if not host_ip: + raise RuntimeError(f"Missing required environment variables: POD_IP={host_ip}") + return host_ip + + +def get_k8s_api(): + from kubernetes import client, config + + kube_config = os.environ.get("KUBECONFIG") + config.load_kube_config(kube_config) + return client.CoreV1Api() + + +# Query ConfigMap from Kubernetes +def query_configmap(name, namespace): + """Query ConfigMap from Kubernetes. + + Args: + name (str): ConfigMap name. + namespace (str): Kubernetes namespace. + + Returns: + V1ConfigMap: ConfigMap object, or None if failed. + """ + from kubernetes.client.rest import ApiException + + k8s_api = get_k8s_api() + try: + configmap = k8s_api.read_namespaced_config_map(name, namespace) + logger.info(f"Successfully queried ConfigMap {name} in namespace {namespace}") + return configmap + except ApiException as e: + logger.error(f"Failed to query ConfigMap {name} in namespace {namespace}: {e}") + return None + except Exception as e: + logger.error(f"Unexpected error querying ConfigMap: {e}") + return None + + +# Get node count from Kubernetes +def discover_worker_nodes(): + """Discover worker nodes from Kubernetes. + + Returns: + int: Number of worker nodes, or 0 if failed. + """ + k8s_api = get_k8s_api() + try: + prefill_pods = k8s_api.list_namespaced_pod( + namespace=NAMESPACE, label_selector="volcano.sh/task-spec=sglang-prefill" + ) + decode_pods = k8s_api.list_namespaced_pod( + namespace=NAMESPACE, label_selector="volcano.sh/task-spec=sglang-decode" + ) + + prefill_count = len(prefill_pods.items) + decode_count = len(decode_pods.items) + nodes_count = prefill_count + decode_count + + logger.info( + f"Discovered {nodes_count} worker nodes (prefill: {prefill_count}, decode: {decode_count})" + ) + return nodes_count + + except Exception as e: + logger.error(f"Unexpected error discovering worker nodes: {e}") + return 0 + + +def set_environment_variables(env_vars): + """Set environment variables. + + Args: + env_vars (dict): Environment variables dictionary. + + Returns: + dict: Updated environment variables. + """ + if not env_vars: + return {} + + for key, value in env_vars.items(): + logger.info(f"Setting ENV_VAR {key}={value}") + os.environ[key] = value + + return env_vars + + +def check_port_availability(host, port, timeout=3): + """Check if the port is available. + + Args: + host (str): Host IP address. + port (int): Port number. + timeout (int): Connection timeout in seconds. + + Returns: + bool: True if port is available, False otherwise. + """ + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(timeout) + result = sock.connect_ex((host, int(port))) + if result == 0: + return True + else: + return False + + except socket.timeout: + logger.error(f"Port check timeout for {host}:{port} after {timeout}s") + return False + except socket.gaierror as e: + logger.error(f"Port check address resolution error for {host}:{port}: {e}") + return False + except socket.error as e: + logger.error(f"Port check socket error for {host}:{port}: {e}") + return False + except ValueError as e: + logger.error(f"Port check invalid value for {host}:{port}: {e}") + return False + except Exception as e: + logger.error(f"Port check unexpected error for {host}:{port}: {e}") + return False + + +def wait_for_all_ports_ready(ips, port, timeout=LOCAL_TIMEOUT, check_interval=15): + """Wait for all nodes' ports to be ready. + + Args: + ips (list): List of IP addresses. + port (int): Port number to check. + timeout (int): Total timeout in seconds. + check_interval (int): Interval between checks in seconds. + + Returns: + bool: True if all ports are ready, False if timeout. + """ + start_time = time.time() + node_status = {ip: False for ip in ips} + + while time.time() - start_time < timeout: + ready_nodes = 0 + status_changed = False + + for ip in ips: + is_ready = check_port_availability(ip, port) + if is_ready != node_status[ip]: + node_status[ip] = is_ready + status_changed = True + if is_ready: + logger.info(f"Node {ip}:{port} is ready") + else: + logger.info(f"Node {ip}:{port} is not ready yet") + if is_ready: + ready_nodes += 1 + + if ready_nodes == len(ips): + logger.info(f"All {len(ips)} nodes' ports are ready!") + return True + + if status_changed: + remaining_nodes = len(ips) - ready_nodes + logger.info(f"Waiting for {remaining_nodes} more nodes to be ready...") + + time.sleep(check_interval) + + logger.info(f"Timeout: Not all nodes are ready after {timeout} seconds") + return False + + +def _get_nnodes_from_args(args_list): + for i, arg in enumerate(args_list): + if arg == "--nnodes" and i + 1 < len(args_list): + return int(args_list[i + 1]) + return None + + +def _get_pod_ip_by_keyword(configmap_data, keyword): + for pod_name, pod_ip in configmap_data.items(): + if keyword in pod_name: + return pod_ip + return None + + +def check_role(allowed_roles: Union[str, Iterable[str]]): + if isinstance(allowed_roles, str): + allowed_roles = {allowed_roles} + else: + allowed_roles = set(allowed_roles) + + if not allowed_roles.issubset(ALL_ROLE_SET): + raise ValueError(f"Invalid allowed roles: {allowed_roles}") + + def decorator(func): + @wraps(func) + def wrapper(self, *args, **kwargs): + current_role = getattr(self, "role", None) + if current_role in allowed_roles: + return func(self, *args, **kwargs) + else: + logger.info( + f"The current node is {current_role}, skip this function {func.__name__}." + ) + return None + + return wrapper + + return decorator + + +# Launch master/worker node +def launch_pd_mix_node(model_config): + logger.info(f"Launch pd mix node start ......") + host_name = get_host_name() + pod_index = int(host_name.rsplit("-", 1)[-1]) + + # Monitor ConfigMap to generate dist-init-addr and node-rank + is_ready = False + dist_init_addr = None + start_time = time.time() + while not is_ready and time.time() - start_time < LOCAL_TIMEOUT: + configmap = query_configmap(CONFIGMAP_NAME, NAMESPACE) + if not configmap or configmap.data is None: + logger.info(f"configmap is None, wait for 15s ......") + time.sleep(15) + continue + logger.info(f"monitor {configmap.data=}") + + master_node_ip = None + for pod_name in configmap.data: + if pod_name.endswith("sglang-node-0"): + master_node_ip = configmap.data[pod_name] + break + if master_node_ip is None: + logger.info(f"Can not find master node in configmap: {configmap.data=}") + time.sleep(15) + continue + + dist_init_addr = f"{master_node_ip}:5000" + logger.info(f"launch_node {dist_init_addr=}") + is_ready = True + + if not is_ready: + raise RuntimeError( + f"Timeout: Failed to get master node information from ConfigMap after {LOCAL_TIMEOUT} seconds" + ) + + special_args = [ + "--dist-init-addr", + dist_init_addr, + "--node-rank", + str(pod_index), + ] + other_args = model_config["other_args"] + for sa in special_args: + other_args.append(sa) + + # if not "--model-type" in other_args: + # other_args += ["--model-type", "llm"] + + for key, value in model_config["node_envs"].items(): + logger.info(f"ENV_VAR_CASE {key}:{value}") + os.environ[key] = value + + host_ip = get_host_ip() + logger.info(f"Starting node, {host_ip=} {other_args=}") + try: + process = popen_launch_server( + model_config["model_path"], + f"http://{host_ip}:{SERVICE_PORT}", + timeout=LOCAL_TIMEOUT, + other_args=[ + *other_args, + ], + ) + except Exception as e: + raise RuntimeError(f"Failed to start node on {host_ip}: {e}") + + return process + + +# Launch prefill/decode separation node +def launch_pd_separation_node(model_config): + logger.info(f"Launch pd separation node start ......") + host_name = get_host_name() + pod_index = int(host_name.rsplit("-", 1)[-1]) + role = "prefill" if "prefill" in host_name else "decode" + + bootstrap_init_port = BOOTSTRAP_INIT_PORT + master_prefill_ip = None + master_decode_ip = None + + is_prefill_instance_multi_node = "--node-rank" not in model_config["prefill_args"] + is_decode_instance_multi_node = "--node-rank" not in model_config["decode_args"] + + # Monitor ConfigMap ready + is_ready = False + start_time = time.time() + configmap_data = None + while not is_ready and time.time() - start_time < LOCAL_TIMEOUT: + configmap = query_configmap(CONFIGMAP_NAME, NAMESPACE) + if not configmap or not configmap.data: + logger.info(f"ConfigMap data is not available yet, waiting for 15s...") + time.sleep(15) + continue + + configmap_data = configmap.data + logger.info(f"Retrieved ConfigMap data: {configmap_data}") + + for pod_name, pod_ip in configmap_data.items(): + if pod_name.endswith("prefill-0"): + master_prefill_ip = pod_ip + if pod_name.endswith("decode-0"): + master_decode_ip = pod_ip + + if master_prefill_ip and master_decode_ip: + is_ready = True + else: + logger.info( + f"Missing master node information - prefill: {master_prefill_ip}, decode: {master_decode_ip}" + ) + logger.info("Retrying in 15s...") + time.sleep(15) + if not is_ready: + raise RuntimeError( + f"Timeout: Failed to get master node information from ConfigMap" + ) + + # Generate prefill/decode run command + service_args = list() + + mf_addr = f"tcp://{master_prefill_ip}:24666" + os.environ["ASCEND_MF_STORE_URL"] = mf_addr + logger.info(f"Setting ENV_VAR ASCEND_MF_STORE_URL={mf_addr}") + + if role == "prefill": + # Current node is prefill + set_environment_variables(model_config.get("prefill_envs")) + + prefill_args = model_config["prefill_args"] + if is_prefill_instance_multi_node: + nnodes = _get_nnodes_from_args(prefill_args) + if nnodes and nnodes > 1: + instance_master_index = (pod_index // nnodes) * nnodes + node_rank = pod_index % nnodes + instance_group_index = pod_index // nnodes + master_pod_keyword = f"prefill-{instance_master_index}" + instance_master_ip = _get_pod_ip_by_keyword( + configmap_data, master_pod_keyword + ) + if not instance_master_ip: + raise RuntimeError( + f"Failed to find instance master {master_pod_keyword} in ConfigMap" + ) + dist_init_addr = f"{instance_master_ip}:5000" + logger.info( + f"Multi-node prefill with nnodes={nnodes}: " + f"pod_index={pod_index}, node_rank={node_rank}, " + f"instance_master={master_pod_keyword}, dist_init_addr={dist_init_addr}" + ) + prefill_args.extend( + [ + "--node-rank", + node_rank, + "--dist-init-addr", + dist_init_addr, + "--disaggregation-bootstrap-port", + str(bootstrap_init_port + instance_group_index), + ] + ) + else: + logger.info( + "No node-rank specified - each prefill node is an independent instance." + ) + prefill_args.extend( + [ + "--node-rank", + 0, + "--disaggregation-bootstrap-port", + str(bootstrap_init_port + pod_index), + ] + ) + else: + logger.info("Node-rank specified - each prefill node is an instance.") + prefill_args.extend( + [ + "--disaggregation-bootstrap-port", + str(bootstrap_init_port + pod_index), + ] + ) + + service_args.extend(prefill_args) + + elif role == "decode": + set_environment_variables(model_config.get("decode_envs")) + + decode_args = model_config["decode_args"] + if is_decode_instance_multi_node: + nnodes = _get_nnodes_from_args(decode_args) + if nnodes and nnodes > 1: + instance_master_index = (pod_index // nnodes) * nnodes + node_rank = pod_index % nnodes + master_pod_keyword = f"decode-{instance_master_index}" + instance_master_ip = _get_pod_ip_by_keyword( + configmap_data, master_pod_keyword + ) + if not instance_master_ip: + raise RuntimeError( + f"Failed to find instance master {master_pod_keyword} in ConfigMap" + ) + dist_init_addr = f"{instance_master_ip}:5000" + logger.info( + f"Multi-node decode with nnodes={nnodes}: " + f"pod_index={pod_index}, node_rank={node_rank}, " + f"instance_master={master_pod_keyword}, dist_init_addr={dist_init_addr}" + ) + decode_args.extend( + [ + "--node-rank", + str(node_rank), + "--dist-init-addr", + dist_init_addr, + ] + ) + else: + logger.info( + "No node-rank specified - each decode node is an independent instance." + ) + decode_args.extend(["--node-rank", "0"]) + else: + logger.info("Node-rank specified - each decode node is an instance.") + + service_args.extend(decode_args) + + host_ip = get_host_ip() + logger.info(f"Starting {role} node on {host_ip} with args: {service_args}") + + other_args = list() + if "--trust-remote-code" not in service_args: + other_args.extend(["--trust-remote-code"]) + if "--attention-backend" not in service_args: + other_args.extend(["--attention-backend", "ascend"]) + if "--device" not in service_args: + other_args.extend(["--device", "npu"]) + if "--disaggregation-transfer-backend" not in service_args: + other_args.extend(["--disaggregation-transfer-backend", "ascend"]) + # if "--model-type" not in service_args: + # other_args.extend(["--model-type", "llm"]) + + other_args.extend(service_args) + + try: + process = popen_launch_server( + model_config["model_path"], + f"http://{host_ip}:{PREFILL_DECODE_PORT}", + timeout=LOCAL_TIMEOUT, + other_args=other_args, + ) + except Exception as e: + raise RuntimeError(f"Failed to start {role} node on {host_ip}: {e}") + + return process + + +# Launch router node +def launch_router(model_config): + logger.info(f"launch_router start ......") + discover_worker_nodes() + + # Monitor to generate prefill/decode URL + prefill_url = [] + decode_url = [] + bootstrap_ports = [] + node_ip_list = [] + is_multi_node_prefill_instance = "--node-rank" not in model_config["prefill_args"] + is_multi_node_decode_instance = "--node-rank" not in model_config["decode_args"] + + prefill_nnodes = ( + _get_nnodes_from_args(model_config["prefill_args"]) + if is_multi_node_prefill_instance + else None + ) + decode_nnodes = ( + _get_nnodes_from_args(model_config["decode_args"]) + if is_multi_node_decode_instance + else None + ) + + is_ready = False + bootstrap_init_port = BOOTSTRAP_INIT_PORT + start_time = time.time() + while not is_ready and time.time() - start_time < ROUTER_CONFIGMAP_TIMEOUT: + configmap = query_configmap(CONFIGMAP_NAME, NAMESPACE) + if not configmap or not configmap.data: + logger.info(f"ConfigMap data is not available yet, waiting for 15s...") + time.sleep(15) + continue + logger.info(f"Retrieved ConfigMap data: {configmap.data}") + + prefill_url.clear() + decode_url.clear() + bootstrap_ports.clear() + node_ip_list.clear() + + for pod_name, pod_ip in configmap.data.items(): + pod_index = int(pod_name.rsplit("-", 1)[-1]) + + if "prefill" in pod_name: + if is_multi_node_prefill_instance: + if prefill_nnodes is not None and prefill_nnodes > 1: + if pod_index % prefill_nnodes == 0: + prefill_url.append(f"{pod_ip}:{PREFILL_DECODE_PORT}") + bootstrap_ports.append( + str(bootstrap_init_port + pod_index // prefill_nnodes) + ) + node_ip_list.append(pod_ip) + elif prefill_nnodes is not None and prefill_nnodes == 1: + prefill_url.append(f"{pod_ip}:{PREFILL_DECODE_PORT}") + bootstrap_ports.append(str(bootstrap_init_port + pod_index)) + node_ip_list.append(pod_ip) + else: + if pod_index == 0: + prefill_url.append(f"{pod_ip}:{PREFILL_DECODE_PORT}") + bootstrap_ports.append(str(bootstrap_init_port)) + node_ip_list.append(pod_ip) + else: + prefill_url.append(f"{pod_ip}:{PREFILL_DECODE_PORT}") + bootstrap_ports.append(str(bootstrap_init_port + pod_index)) + node_ip_list.append(pod_ip) + + if "decode" in pod_name: + if is_multi_node_decode_instance: + if decode_nnodes is not None and decode_nnodes > 1: + if pod_index % decode_nnodes == 0: + decode_url.append(f"{pod_ip}:{PREFILL_DECODE_PORT}") + node_ip_list.append(pod_ip) + elif decode_nnodes is not None and decode_nnodes == 1: + decode_url.append(f"{pod_ip}:{PREFILL_DECODE_PORT}") + node_ip_list.append(pod_ip) + else: + if pod_index == 0: + decode_url.append(f"{pod_ip}:{PREFILL_DECODE_PORT}") + node_ip_list.append(pod_ip) + else: + decode_url.append(f"{pod_ip}:{PREFILL_DECODE_PORT}") + node_ip_list.append(pod_ip) + + if prefill_url and decode_url: + is_ready = True + else: + logger.info("Incomplete node information in ConfigMap, waiting for 15s...") + time.sleep(15) + + if not is_ready: + raise RuntimeError( + f"Timeout: Failed to get complete node information from ConfigMap" + ) + logger.info( + f"ConfigMap monitoring complete: prefill_url={prefill_url}, decode_url={decode_url}, " + f"bootstrap_ports={bootstrap_ports}, node_ip_list={node_ip_list}" + ) + + # Check all node port ready + if not wait_for_all_ports_ready( + ips=node_ip_list, port=PREFILL_DECODE_PORT, timeout=LOCAL_TIMEOUT + ): + raise RuntimeError("Failed to wait for all nodes to be ready") + + # Set environment variables + set_environment_variables(model_config.get("router_envs")) + + router_args = model_config["router_args"] + # Router server params + router_command = [ + "python3", + "-u", + "-m", + "sglang_router.launch_router", + "--host", + "0.0.0.0", + "--port", + str(SERVICE_PORT), + "--pd-disaggregation", + "--policy", + "cache_aware", + *[str(x) for x in router_args], + ] + + for index, url in enumerate(prefill_url): + router_command.extend( + ["--prefill", f"http://{url}", f"{bootstrap_ports[index]}"] + ) + + for url in decode_url: + router_command.extend(["--decode", f"http://{url}"]) + + logger.info(f"Starting router with command: {' '.join(router_command)}") + try: + router_process = subprocess.Popen(router_command) + logger.info(f"Router process started with PID: {router_process.pid}") + except Exception as e: + raise RuntimeError(f"Failed to start router process: {e}") + + +def wait_server_ready(url, timeout=LOCAL_TIMEOUT): + """Wait for the server to be ready. + + Args: + url (str): Server URL to check. + timeout (int): Timeout in seconds. + + Raises: + RuntimeError: If server fails to start within timeout. + """ + logger.info(f"Waiting for the server to start at {url}...") + start_time = time.perf_counter() + check_interval = 10 + + while True: + try: + response = requests.get(url, timeout=30) + if response.status_code == 200: + logger.info(f"Server {url} is ready!") + return + else: + logger.info( + f"Server {url} returned status code: {response.status_code}" + ) + except Exception as e: + # logger.error(f"Server {url} request error: {e}, retrying...") + pass + + elapsed_time = time.perf_counter() - start_time + if elapsed_time > timeout: + raise RuntimeError( + f"Server {url} failed to start in {timeout}s (elapsed: {elapsed_time:.2f}s)" + ) + time.sleep(check_interval) + + +class TestNpuMultiNodePdMixTestCaseBase(CustomTestCase): + model_config = None + + @classmethod + def setUpClass(cls): + cls.process = None + cls.local_ip = "127.0.0.1" + cls.host = os.getenv("POD_IP") + cls.port = SERVICE_PORT + cls.base_url = f"http://{cls.host}:{cls.port}" + cls.hostname = os.getenv("HOSTNAME") + cls.role = "master" if cls.hostname.endswith("sglang-node-0") else "worker" + logger.info(f"Init {cls.host} {cls.role=}!") + cls.sglang_thread = None + cls.stop_event = threading.Event() + + @classmethod + def tearDownClass(cls): + if cls.process: + try: + kill_process_tree(cls.process.pid) + except Exception as e: + logger.error(f"Error during tearDown: {e}") + + @classmethod + @check_role(allowed_roles=["master"]) + def launch_pd_mix_master_node(cls): + logger.info(f"Starting master node in thread...") + cls.sglang_thread = threading.Thread( + target=launch_pd_mix_node, args=(cls.model_config,) + ) + cls.sglang_thread.daemon = True + cls.sglang_thread.start() + + health_check_url = f"{cls.base_url}/health" + logger.info(f"Waiting for router to be ready at {health_check_url}") + wait_server_ready(health_check_url) + + logger.info( + f"Waiting {SERVER_INITIALIZATION_DELAY} seconds for the server to fully initialize..." + ) + time.sleep(SERVER_INITIALIZATION_DELAY) + + @classmethod + @check_role(allowed_roles=["worker"]) + def launch_pd_mix_worker_node(cls): + logger.info(f"Starting master node in thread...") + cls.sglang_thread = threading.Thread( + target=launch_pd_mix_node, args=(cls.model_config,) + ) + cls.sglang_thread.daemon = True + cls.sglang_thread.start() + keep_alive_time = 1800 + logger.info( + f"{cls.role} node started, keeping test alive for {keep_alive_time} seconds" + ) + time.sleep(keep_alive_time) + + @classmethod + @check_role(allowed_roles=["master", "worker"]) + def stop_sglang_thread(cls): + if cls.sglang_thread: + logger.info(f"Stopping sglang thread {cls.sglang_thread}") + if cls.sglang_thread.is_alive(): + logger.info("Notifying stop event...") + cls.stop_event.set() + cls.sglang_thread.join(timeout=5) + if cls.sglang_thread.is_alive(): + logger.info( + "Warning: subprocess is not terminated normally, it may has been already force stopped." + ) + else: + logger.info("Subprocess has been Stopped.") + else: + logger.info("No running sglang thread.") + + @check_role(allowed_roles=["master"]) + def run_gsm8k_test( + self, + expect_accuracy, + num_shots=8, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + ): + args = SimpleNamespace( + num_shots=num_shots, + data_path=data_path, + num_questions=num_questions, + max_new_tokens=max_new_tokens, + parallel=parallel, + host=f"http://{self.host}", + port=self.port, + ) + logger.info("Starting gsm8k test...") + metrics = run_eval_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + expect_accuracy, + f'Accuracy is {str(metrics["accuracy"])}, is lower than {expect_accuracy}', + ) + + +class TestNpuMultiNodePdSepTestCaseBase(CustomTestCase): + model_config = None + + @classmethod + def setUpClass(cls): + cls.process = None + cls.local_ip = "127.0.0.1" + cls.host = os.getenv("POD_IP") + cls.port = SERVICE_PORT + cls.base_url = f"http://{cls.host}:{cls.port}" + cls.hostname = os.getenv("HOSTNAME") + cls.role = ( + "router" + if "router" in cls.hostname + else "prefill" if "prefill" in cls.hostname else "decode" + ) + logger.info(f"Init {cls.host} {cls.role=}!") + cls.sglang_thread = None + cls.stop_event = threading.Event() + + @classmethod + def tearDownClass(cls): + if cls.process: + try: + kill_process_tree(cls.process.pid) + except Exception as e: + logger.error(f"Error during tearDown: {e}") + + @classmethod + @check_role(allowed_roles=["router"]) + def start_router_server(cls): + logger.info(f"Starting router in thread...") + cls.sglang_thread = threading.Thread( + target=launch_router, args=(cls.model_config,) + ) + cls.sglang_thread.daemon = True + cls.sglang_thread.start() + + health_check_url = f"{cls.base_url}/health" + logger.info(f"Waiting for router to be ready at {health_check_url}") + wait_server_ready(health_check_url) + + logger.info( + f"Waiting {SERVER_INITIALIZATION_DELAY} seconds for the server to fully initialize..." + ) + time.sleep(SERVER_INITIALIZATION_DELAY) + + @classmethod + @check_role(allowed_roles=["prefill", "decode"]) + def start_pd_server(cls): + logger.info(f"Starting pd separation node in thread...") + cls.sglang_thread = threading.Thread( + target=launch_pd_separation_node, args=(cls.model_config,) + ) + cls.sglang_thread.daemon = True + cls.sglang_thread.start() + keep_alive_time = 1800 + logger.info( + f"{cls.role} node started, keeping test alive for {keep_alive_time} seconds" + ) + time.sleep(keep_alive_time) + + @classmethod + @check_role(allowed_roles=["prefill", "decode", "router"]) + def stop_sglang_thread(cls): + if cls.sglang_thread: + logger.info(f"Stopping sglang thread {cls.sglang_thread}") + if cls.sglang_thread.is_alive(): + logger.info("Notifying stop event...") + cls.stop_event.set() + cls.sglang_thread.join(timeout=5) + if cls.sglang_thread.is_alive(): + logger.info( + "Warning: subprocess is not terminated normally, it may has been already force stopped." + ) + else: + logger.info("Subprocess has been Stopped.") + else: + logger.info("No running sglang thread.") + + @check_role(allowed_roles=["router"]) + def run_gsm8k_test( + self, + expect_accuracy, + num_shots=8, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + ): + args = SimpleNamespace( + num_shots=num_shots, + data_path=data_path, + num_questions=num_questions, + max_new_tokens=max_new_tokens, + parallel=parallel, + host=f"http://{self.host}", + port=self.port, + ) + logger.info("Starting gsm8k test...") + metrics = run_eval_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + expect_accuracy, + f'Accuracy is {str(metrics["accuracy"])}, is lower than {expect_accuracy}', + ) diff --git a/python/sglang/test/ascend/e2e/test_npu_performance_utils.py b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py new file mode 100644 index 000000000..716945498 --- /dev/null +++ b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py @@ -0,0 +1,1230 @@ +import logging +import os +import re +import subprocess +import threading +import time +from functools import wraps +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.e2e.gen_dataset_fixed_len import ( + generate_gsm8k_dataset, + generate_mm_dataset, + generate_random_dataset, + save_jsonl, +) +from sglang.test.ascend.e2e.test_npu_multi_node_utils import ( + SERVICE_PORT, + check_role, + launch_pd_mix_node, + launch_pd_separation_node, + launch_router, + wait_server_ready, +) +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + dump_metric, + popen_launch_server, +) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", + handlers=[logging.StreamHandler()], +) +logger = logging.getLogger(__name__) + +AISBENCHMARK = "aisbench" +BENCHSERVING = "bench-serving" +BENCHMARK_TOOL_DEFAULT = BENCHSERVING +AISBENCHMARK_DATASET_GSM8K = "gsm8k" +AISBENCHMARK_DATASET_SHAREGPT = "sharegpt" +AISBENCHMARK_DATASET_MM_CUSTOM_GEN = "mm-custom-gen" +AISBENCHMARK_DATASET_DEFAULT = AISBENCHMARK_DATASET_GSM8K + +SHAREGPT_DATASET_TEST_FILE = "/tmp/ShareGPT_V3_unfiltered_cleaned_split.json" +GSM8K_DATASET_TEST_FILE = ( + "/root/.cache/modelscope/hub/datasets/grade_school_math/test.jsonl" +) +GSM8K_DATASET_TRAIN_FILE = ( + "/root/.cache/modelscope/hub/datasets/grade_school_math/train.jsonl" +) + +PYTHON_FOR_TEST_TOOL = "python_venv_for_test_tool/bin/python" +if not os.path.exists(PYTHON_FOR_TEST_TOOL) or not os.access( + PYTHON_FOR_TEST_TOOL, os.X_OK +): + PYTHON_FOR_TEST_TOOL = "python3" +logger.info(f"PYTHON_FOR_TEST_TOOL: {PYTHON_FOR_TEST_TOOL}") + +DEEPSEEK_R1_W8A8_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Howeee/DeepSeek-R1-0528-w8a8" +) +DEEPSEEK_R1_W4A8_PER_CHANNEL_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/DeepSeek-R1-0528-w4a8-per-channel" +) +DEEPSEEK_V32_W8A8_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/vllm-ascend/DeepSeek-V3.2-W8A8" +) +QWEN3_8B_W8A8_MODEL_PATH = "/root/.cache/modelscope/hub/models/Qwen/Qwen3-8B-W8A8" +QWEN3_8B_EAGLE_MODEL_PATH = "/root/.cache/modelscope/hub/models/Qwen/Eagle3-Qwen3-8B-zh" +QWEN3_14B_MODEL_PATH = "/root/.cache/modelscope/hub/models/Qwen/Qwen3-14B" +QWEN3_14B_LORA_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Qwen/Qwen3-14B-Lora/Qwen3-14B_lora" +) +QWEN3_14B_W8A8_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Qwen/Qwen3-14B-W8A8-Dynamic2" +) +QWEN3_14B_EAGLE_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/AngelSlim/Qwen3-14B_eagle3" +) +QWEN3_5_27B_MODEL_PATH = "/root/.cache/modelscope/hub/models/Qwen/Qwen3.5-27B" +QWEN3_5_27B_W8A8_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Eco-Tech/Qwen3.5-27B-W8A8" +) +QWEN3_30B_A3B_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Qwen/Qwen3-30B-A3B-Instruct-2507" +) +QWEN3_6_35B_A3B_MODEL_PATH = "/root/.cache/modelscope/hub/models/Qwen/Qwen3.6-35B-A3B" +QWEN3_6_27B_MODEL_PATH = "/root/.cache/modelscope/hub/models/Qwen/Qwen3.6-27B" +QWEN3_6_27B_W8A8_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Eco-Tech/Qwen3.6-27B-w8a8" +) +QWEN3_30B_A3B_W8A8_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Qwen/Qwen3-30B-A3B-w8a8" +) +QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/vllm-ascend/Qwen3-30B-A3B-W8A8" +) +QWEN3_A3B_EAGLE_MODEL_PATH = "/root/.cache/modelscope/hub/models/Qwen/Qwen3-a3B_eagle3" +QWEN3_32B_MODEL_PATH = "/root/.cache/modelscope/hub/models/Qwen/Qwen3-32B" +QWEN3_32B_W8A8_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/aleoyang/Qwen3-32B-w8a8-MindIE" +) +QWEN3_32B_EAGLE_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Zjcxy-SmartAI/Eagle3-Qwen3-32B-zh" +) +QWEN3_235B_MODEL_PATH = "/root/.cache/modelscope/hub/models/Qwen/Qwen3-235B-A22B" +QWEN3_235B_W8A8_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/vllm-ascend/Qwen3-235B-A22B-W8A8" +) +QWEN3_235B_A22B_EAGLE_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Qwen/Qwen3-235B-A22B-Eagle3" +) +QWEN3_235B_A22B_INSTRUCT_2507_W8A8_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/zcgy26/Qwen3-235B-A22B-Instruct-2507-w8a8" +) +QWEN3_480B_W8A8_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Qwen3-Coder-480B-A35B-Instruct-w8a8-QuaRot" +) +QWEN3_NEXT_80B_A3B_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Qwen/Qwen3-Next-80B-A3B-Instruct" +) +QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/vllm-ascend/Qwen3-Next-80B-A3B-Instruct-W8A8" +) +QWEN3_CODER_NEXT_W8A8_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Qwen/Qwen3-Coder-Next-W8A8" +) +GLM_4_6_W8A8_MODEL_PATH = "/root/.cache/modelscope/hub/models/GLM-4.6-w8a8_WITH_MTP" + +QWEN3_VL_8B_MODEL_PATH = "/root/.cache/modelscope/hub/models/Qwen/Qwen3-VL-8B-Instruct" +QWEN3_VL_30B_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Qwen/Qwen3-VL-30B-A3B-Instruct" +) +QWEN3_VL_235B_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Qwen/Qwen3-VL-235B-A22B-Instruct" +) +QWEN2_5_VL_72B_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Qwen/Qwen2.5-VL-72B-Instruct-w8a8" +) +KIMI_K2_5_W4A8_MODEL_PATH = "/root/.cache/modelscope/hub/models/Eco-Tech/Kimi-K2.5-w4a8" +KIMI_K2_5_EAGLE3_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/lightseekorg/kimi-k2.5-eagle3" +) +GLM_4_7_FLASH_MODEL_PATH = "/root/.cache/modelscope/hub/models/ZhipuAI/GLM-4.7-Flash" +GLM_5_1_W4A8_MODEL_PATH = "/root/.cache/modelscope/hub/models/Eco-Tech/GLM-5.1-w4a8" +MINIMAX_M2_5_W8A8_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Eco-Tech/MiniMax-M2.5-w8a8-QuaRot" +) +MINIMAX_M2_5_EAGLE3_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/sgl-npu/MiniMax-M2.5-eagel-model-0318" +) + +QWEN3_5_397B_W8A8_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp" +) +QWEN3_5_397B_W4A8_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Eco-Tech/Qwen3.5-397B-A17B-w4a8-mtp" +) +KIMI_K2_6_W4A8_MODEL_PATH = "/root/.cache/modelscope/hub/models/Eco-Tech/Kimi-K2.6-w4a8" +KIMI_K2_6_EAGLE3_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/lightseekorg/kimi-k2.6-eagle3" +) +GLM_4_6V_FLASH_MODEL_PATH = "/root/.cache/modelscope/hub/models/ZhipuAI/GLM-4.6V-Flash" +QWEN3_VL_8B_THINKING_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Qwen/Qwen3-VL-8B-Thinking" +) +QWEN3_VL_30B_A3B_THINKING_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Qwen/Qwen3-VL-30B-A3B-Thinking" +) +QWEN3_OMNI_30B_A3B_THINKING_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Qwen/Qwen3-Omni-30B-A3B-Thinking" +) +ROUND_ROBIN = "round_robin" + +DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 3600 +MAX_SERVER_KEEP_ALIVE_TIME = 3600 + +# Timeouts and delays +SERVER_INITIALIZATION_DELAY = 120 + +# Test parameters +PROMPTS_MULTIPLIER = 4 + +# Metrics thresholds +TPOT_THRESHOLD = 50 +TPOT_TOLERANCE_LOW = 1.0 # +1 second +TPOT_TOLERANCE_HIGH = 1.02 # +2% +TTFT_TOLERANCE = 1.02 # +2% +E2E_TOLERANCE = 1.02 # +2% +OUTPUT_TOKEN_THROUGHPUT_TOLERANCE = 0.98 # -2% + +# Package filtering keywords +PACKAGE_FILTER_KEYWORDS = [ + "sglang", + "sgl", + "torch", + "deep-ep", + "memfabric_hybrid", +] + +if os.environ.get("ASCEND_RT_VISIBLE_DEVICES"): + DEFAULT_SERVER_PORT_FOR_TEST = ( + 20000 + int(os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "0")[0]) * 100 + ) +else: + DEFAULT_SERVER_PORT_FOR_TEST = ( + 20000 + int(os.environ.get("ASCEND_VISIBLE_DEVICES", "0")[0]) * 100 + ) +DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_SERVER_PORT_FOR_TEST + 66}" + + +def retry(max_attempts: int = None): + """ + Test case retry decorator + Args: + max_attempts (int): Maximum number of execution attempts. If None, use self.max_attempts. + """ + + def decorator(func): + @wraps(func) + def wrapper(self, *args, **kwargs): + # Store the last exception for final reporting + last_exception = None + + # Get max_attempts from instance if not provided in decorator + attempts = max_attempts or getattr(self, "max_attempts", 2) + + # Execute the test up to max_attempts times + for attempt in range(1, attempts + 1): + try: + logger.info(f"Executing test attempt {attempt}/{attempts}") + return func( + self, *args, **kwargs + ) # Return immediately if test passes + except (AssertionError, Exception) as e: + last_exception = e + logger.info(f"Test failed on attempt {attempt}") + + # Raise the last exception if all attempts failed + raise last_exception + + return wrapper + + return decorator + + +def get_cann_version(): + """Get CANN version info. + + Returns: + str: CANN version info string. + """ + cann_info_file = "/usr/local/Ascend/ascend-toolkit/latest/aarch64-linux/ascend_toolkit_install.info" + cann_ver_num = None + + try: + with open(cann_info_file, "r", encoding="utf-8") as f: + for line in f: + if line.startswith("version="): + cann_ver_num = line.strip().split("=")[-1] + break + + if cann_ver_num: + cann_version_info = f"CANN: {cann_ver_num}" + logger.info(cann_version_info) + return cann_version_info + else: + logger.info("CANN version not found") + return f"CANN: {cann_ver_num}" + + except FileNotFoundError: + logger.error(f"CANN info file not found: {cann_info_file}") + return f"CANN: {cann_ver_num}" + except Exception as e: + logger.error(f"Error reading CANN info: {e}") + return f"CANN: {cann_ver_num}" + + +def write_pkg_info_to_file(result_file): + """Write package information to result file. + + Args: + result_file (str): Path to the result file. + """ + import transformers + + try: + pip_output = subprocess.run( + ["pip", "list"], capture_output=True, text=True, check=False + ) + packages = pip_output.stdout + + # Filter relevant packages using list comprehension + filtered_packages = [ + line + for line in packages.split("\n") + if any(keyword in line for keyword in PACKAGE_FILTER_KEYWORDS) + ] + + # Write to result file + os.makedirs(os.path.dirname(os.path.abspath(result_file)), exist_ok=True) + with open(result_file, "w", encoding="utf-8") as f: + for pkg in filtered_packages: + f.write(pkg + "\n") + logger.info(pkg) + f.write(get_cann_version() + "\n") + transformers_version_info = ( + "transformers: " + transformers.__version__ + "\n" + ) + f.write(transformers_version_info) + logger.info(transformers_version_info) + + except Exception as e: + logger.error(f"Error getting packages: {e}") + + +def run_bench_serving( + host, + port, + model_path=None, + backend="sglang", + dataset_name=None, + dataset_path=None, + request_rate=None, + max_concurrency=None, + num_prompts=None, + input_len=None, + output_len=None, + random_range_ratio=1, + image_resolution=None, + image_count=None, + warmup_requests=None, + seed=None, + output_file=None, + repeat_rate=None, + temperature=None, + top_p=None, +): + metrics_path = os.getenv("METRICS_DATA_FILE") + result_file = ( + "./bench_log.txt" + if not metrics_path + else f"{metrics_path}/bench_serving_metrics.txt" + ) + logger.info(f"The metrics result file: {result_file}") + + write_pkg_info_to_file(result_file) + + if dataset_name == "generated-shared-prefix": + cmd_args = [ + PYTHON_FOR_TEST_TOOL, + "-m", + "sglang.bench_serving", + "--host", + host, + "--port", + str(port), + "--model", + model_path, + "--backend", + backend, + "--dataset-name", + dataset_name, + "--gsp-num-groups", + "1", + "--gsp-prompts-per-group", + str(num_prompts), + "--gsp-system-prompt-len", + ( + str(int((repeat_rate if repeat_rate is not None else 0.9) * input_len)) + if input_len + else "0" + ), + "--gsp-question-len", + ( + str( + int( + (1 - (repeat_rate if repeat_rate is not None else 0.9)) + * input_len + ) + ) + if input_len + else "0" + ), + "--gsp-output-len", + str(output_len) if output_len else "0", + ] + if max_concurrency: + cmd_args.extend(["--max-concurrency", str(max_concurrency)]) + if num_prompts: + cmd_args.extend(["--num-prompts", str(num_prompts)]) + if request_rate: + cmd_args.extend(["--request-rate", str(request_rate)]) + if temperature is not None: + cmd_args.extend(["--temperature", str(temperature)]) + if top_p is not None: + cmd_args.extend(["--top-p", str(top_p)]) + else: + cmd_args = [ + PYTHON_FOR_TEST_TOOL, + "-m", + "sglang.bench_serving", + "--host", + host, + "--port", + str(port), + "--model", + model_path, + "--backend", + backend, + ] + + if dataset_name: + cmd_args.extend(["--dataset-name", str(dataset_name)]) + if dataset_path: + cmd_args.extend(["--dataset-path", str(dataset_path)]) + if request_rate: + cmd_args.extend(["--request-rate", str(request_rate)]) + if max_concurrency: + cmd_args.extend(["--max-concurrency", str(max_concurrency)]) + if num_prompts: + cmd_args.extend(["--num-prompts", str(num_prompts)]) + if input_len: + cmd_args.extend(["--random-input-len", str(input_len)]) + if output_len: + cmd_args.extend(["--random-output-len", str(output_len)]) + if random_range_ratio: + cmd_args.extend(["--random-range-ratio", str(random_range_ratio)]) + if image_resolution: + cmd_args.extend(["--image-resolution", str(image_resolution)]) + if image_count: + cmd_args.extend(["--image-count", str(image_count)]) + if warmup_requests: + cmd_args.extend(["--warmup-requests", str(warmup_requests)]) + if seed: + cmd_args.extend(["--seed", str(seed)]) + if output_file: + cmd_args.extend(["--output-file", str(output_file)]) + if temperature is not None: + cmd_args.extend(["--temperature", str(temperature)]) + if top_p is not None: + cmd_args.extend(["--top-p", str(top_p)]) + logger.info(f"Command: {' '.join(cmd_args)}") + + # Run benchmark command and capture output + metrics = {"mean_ttft": None, "mean_tpot": None, "total_tps": None} + + process = subprocess.Popen( + cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1 + ) + try: + # Read output line by line + with open(result_file, "a", encoding="utf-8") as f: + for line in process.stdout: + if line.strip(): + print(line, end="") + f.write(line) + stripped_line = line.strip() + + # Extract metrics + if "Mean TTFT" in stripped_line: + parts = stripped_line.split() + if len(parts) >= 4: + metrics["mean_ttft"] = parts[3] + elif "Mean TPOT" in stripped_line: + parts = stripped_line.split() + if len(parts) >= 4: + metrics["mean_tpot"] = parts[3] + elif "Output token throughput" in stripped_line: + parts = stripped_line.split() + if len(parts) >= 5: + metrics["total_tps"] = parts[4] + elif "Mean E2E Latency" in stripped_line: + parts = stripped_line.split() + if len(parts) >= 5: + metrics["mean_e2e_latency"] = parts[4] + process.wait() + if process.returncode != 0: + logger.error( + f"Benchmark command failed with return code: {process.returncode}" + ) + except Exception as e: + logger.error(f"Error running benchmark: {e}") + finally: + if process.stdout is not None and not process.stdout.closed: + process.stdout.close() + + return metrics + + +def run_aisbench( + host, + port, + model_path, + dataset_type, + dataset_path, + input_len, + output_len, + max_concurrency, + num_prompts, + image_resolution=None, + random_range_ratio=1, + request_rate=None, + repeat_rate=None, + dp=None, + generation_kwargs=None, +): + + if dataset_type == "sharegpt": + dataset_file = f"/tmp/datasets/test.jsonl" + if not os.path.exists(dataset_file): + logger.info( + f"Generating random dataset from ShareGPT: {dataset_file}, " + f"model_path={model_path}, batch_size={num_prompts}, input_len={input_len}" + ) + generate_random_dataset( + model_path=model_path, + source_dataset_path=SHAREGPT_DATASET_TEST_FILE, + batch_size=num_prompts, + input_len=input_len, + output_file=dataset_file, + output_len=output_len, + range_ratio=random_range_ratio, + ) + dataset_path = dataset_file + logger.info(f"Dataset generated: {dataset_path}") + + elif dataset_type == AISBENCHMARK_DATASET_GSM8K and not dataset_path: + dataset_file = f"/tmp/datasets/test.jsonl" + if not os.path.exists(dataset_file): + logger.info( + f"Generating gsm8k dataset: {dataset_file}, " + f"model_path={model_path}, batch_size={num_prompts}, input_len={input_len}" + ) + generate_gsm8k_dataset( + model_path=model_path, + source_dataset_path=GSM8K_DATASET_TEST_FILE, + batch_size=num_prompts, + input_len=input_len, + output_file=dataset_file, + ) + dataset_path = dataset_file + logger.info(f"Dataset generated: {dataset_path}") + + elif dataset_type == AISBENCHMARK_DATASET_MM_CUSTOM_GEN and not dataset_path: + dataset_file = f"/tmp/datasets/mm.jsonl" + if not os.path.exists(dataset_file): + image_dir = f"/tmp/datasets/images" + data = generate_mm_dataset( + train_path=GSM8K_DATASET_TRAIN_FILE, + test_path=GSM8K_DATASET_TEST_FILE, + tokenizer_path=model_path, + target_tokens=input_len, + num_prompts=num_prompts, + image_dir=image_dir, + size=image_resolution, + trust_remote_code=True, + ) + save_jsonl(data, dataset_file) + dataset_path = dataset_file + logger.info(f"Dataset generated: {dataset_file}") + + else: + logger.info(f"Use exist dataset: {dataset_path}") + + metrics_path = os.getenv("METRICS_DATA_FILE") + result_path = "./aisbench_result" if not metrics_path else metrics_path + logger.info(f"The metrics result file: {result_path}") + + cmd = f"/bin/bash /root/sglang/python/sglang/test/ascend/e2e/run_aisbench.sh " + cmd += f"--mode perf " + cmd += f"--ip {host} " + cmd += f"--port {str(port)} " + cmd += f"--model {os.path.basename(model_path)} " + cmd += f"--model-path {model_path} " + cmd += f"--dataset-type {dataset_type} " + cmd += f"--dataset-path {dataset_path} " + cmd += f"--input-len {str(input_len)} " + cmd += f"--output-len {str(output_len)} " + cmd += f"--batch-size {str(max_concurrency)} " + cmd += f"--num-prompts {str(num_prompts)} " + cmd += f"--output-path {result_path}" + + if request_rate is not None: + cmd += f" --request_rate {request_rate}" + if repeat_rate is not None: + cmd += f" --repeat_rate {repeat_rate}" + if dp is not None: + cmd += f" --dp {dp}" + if generation_kwargs: + cmd += f" --generation-kwargs '{generation_kwargs}'" + + logger.info(f"Command: {cmd}") + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + shell=True, + ) + + output_lines = [] + try: + for line in iter(process.stdout.readline, ""): + if line.strip(): + print(line, end="") + output_lines.append(line.strip()) + + process.wait() + + if process.returncode != 0: + logger.error(f"Command failed with return code: {process.returncode}") + raise subprocess.CalledProcessError(process.returncode, cmd) + + logger.info("Command executed successfully") + + metrics = {} + full_output = "\n".join(output_lines) + + simplified_output = re.sub(r"[^\w\s.]", " ", full_output) + + tpot_match = re.search(r"TPOT\s+total\s+([\d.]+)\s+ms", simplified_output) + if tpot_match: + metrics["mean_tpot"] = tpot_match.group(1) + logger.info(f"Extracted mean_tpot: {metrics['mean_tpot']} ms") + else: + logger.warning("Could not extract mean_tpot from output") + logger.error( + f"Simplified output snippet around TPOT: {simplified_output[simplified_output.find('TPOT')-20:simplified_output.find('TPOT')+50] if 'TPOT' in simplified_output else 'TPOT not found'}" + ) + + tps_matches = re.findall( + r"Output\s+Token\s+Throughput\s+total\s+([\d.]+)\s+token\s*/?\s*s", + simplified_output, + ) + if len(tps_matches) < 2: + tps_matches += re.findall( + r"OutputTokenThroughput\s+total\s+([\d.]+)\s+token\s*/?\s*s", + simplified_output, + ) + + logger.info( + f"Found {len(tps_matches)} matches for Output Token Throughput: {tps_matches}" + ) + if tps_matches: + # The first match is from the Common Metric section, which is the total throughput + metrics["total_tps"] = tps_matches[0] + if len(tps_matches) >= 2: + logger.info( + f"Extracted total_tps: {metrics['total_tps']} token/s (from Common Metric section)" + ) + else: + logger.info( + f"Extracted total_tps: {metrics['total_tps']} token/s (only one match found)" + ) + else: + logger.warning("Could not extract total_tps from output") + logger.warning( + f"Simplified output snippet around Output Token Throughput: {simplified_output[simplified_output.find('Output')-20:simplified_output.find('Output')+100] if 'Output' in simplified_output else 'Output not found'}" + ) + + ttft_match = re.search(r"TTFT\s+total\s+([\d.]+)\s+ms", simplified_output) + if ttft_match: + metrics["mean_ttft"] = ttft_match.group(1) + logger.info(f"Extracted mean_ttft: {metrics['mean_ttft']} ms") + else: + logger.warning("Could not extract mean_ttft from output") + logger.warning( + f"Simplified output snippet around TTFT: {simplified_output[simplified_output.find('TTFT')-20:simplified_output.find('TTFT')+50] if 'TTFT' in simplified_output else 'TTFT not found'}" + ) + + e2el_match = re.search(r"E2EL\s+total\s+([\d.]+)\s+ms", simplified_output) + if e2el_match: + metrics["mean_e2e_latency"] = e2el_match.group(1) + logger.info(f"Extracted mean_e2e_latency: {metrics['mean_e2e_latency']} ms") + else: + logger.warning("Could not extract mean_e2e_latency from output") + logger.warning( + f"Simplified output snippet around E2EL: {simplified_output[simplified_output.find('E2EL')-20:simplified_output.find('E2EL')+50] if 'E2EL' in simplified_output else 'E2EL not found'}" + ) + + concurrency_match = re.search( + r"Concurrency\s+total\s+([\d.]+)", simplified_output + ) + if concurrency_match: + metrics["concurrency"] = concurrency_match.group(1) + logger.info(f"Extracted concurrency: {metrics['concurrency']}") + else: + logger.warning("Could not extract concurrency from output") + logger.warning( + f"Simplified output snippet around Concurrency: {simplified_output[simplified_output.find('Concurrency')-20:simplified_output.find('Concurrency')+50] if 'Concurrency' in simplified_output else 'Concurrency not found'}" + ) + + max_concurrency_match = re.search( + r"Max\s+Concurrency\s+total\s+([\d.]+)", simplified_output + ) + if max_concurrency_match: + metrics["max_concurrency"] = max_concurrency_match.group(1) + logger.info(f"Extracted max_concurrency: {metrics['max_concurrency']}") + else: + logger.warning("Could not extract max_concurrency from output") + logger.warning( + f"Simplified output snippet around Max Concurrency: {simplified_output[simplified_output.find('Max Concurrency')-20:simplified_output.find('Max Concurrency')+50] if 'Max Concurrency' in simplified_output else 'Max Concurrency not found'}" + ) + + req_throughput_match = re.search( + r"Request\s+Throughput\s+total\s+([\d.]+)\s+req\s*/?\s*s", + simplified_output, + ) + if req_throughput_match: + metrics["request_throughput"] = req_throughput_match.group(1) + logger.info( + f"Extracted request_throughput: {metrics['request_throughput']} req/s" + ) + else: + logger.warning("Could not extract request_throughput from output") + logger.warning( + f"Simplified output snippet around Request Throughput: {simplified_output[simplified_output.find('Request')-20:simplified_output.find('Request')+50] if 'Request' in simplified_output else 'Request not found'}" + ) + + total_requests_match = re.search( + r"Total\s+Requests\s+total\s+(\d+)", simplified_output + ) + if total_requests_match: + metrics["total_requests"] = total_requests_match.group(1) + logger.info(f"Extracted total_requests: {metrics['total_requests']}") + else: + logger.warning("Could not extract total_requests from output") + logger.warning( + f"Simplified output snippet around Total Requests: {simplified_output[simplified_output.find('Total Requests')-20:simplified_output.find('Total Requests')+50] if 'Total Requests' in simplified_output else 'Total Requests not found'}" + ) + + failed_requests_match = re.search( + r"Failed\s+Requests\s+total\s+(\d+)", simplified_output + ) + if failed_requests_match: + metrics["failed_requests"] = failed_requests_match.group(1) + logger.info(f"Extracted failed_requests: {metrics['failed_requests']}") + else: + logger.warning("Could not extract failed_requests from output") + logger.warning( + f"Simplified output snippet around Failed Requests: {simplified_output[simplified_output.find('Failed Requests')-20:simplified_output.find('Failed Requests')+50] if 'Failed Requests' in simplified_output else 'Failed Requests not found'}" + ) + + logger.info(f"All extracted metrics: {metrics}") + + return metrics + + except KeyboardInterrupt: + logger.info("Keyboard interrupt received, terminating process...") + process.terminate() + try: + process.wait(timeout=5) + logger.info("Process terminated") + except subprocess.TimeoutExpired: + logger.warning("Process did not terminate gracefully, killing it...") + process.kill() + logger.info("Process killed") + raise + except Exception as e: + logger.error(f"Error executing command: {e}") + process.terminate() + process.wait(timeout=5) + raise + + +def assert_metrics(self, metrics): + """Assert benchmark metrics against expected values. + + Args: + metrics (dict): Benchmark metrics dictionary. + """ + if not metrics: + raise Exception("No metrics obtained from benchmark") + + tc_name = self.__class__.__name__ + if self.tpot and metrics.get("mean_tpot"): + dump_metric( + "tpot", + float(metrics["mean_tpot"]), + labels={"test_case": tc_name, "type": "perf"}, + ) + dump_metric( + "tpot_baseline", + float(self.tpot), + labels={"test_case": tc_name, "type": "perf"}, + ) + if self.output_token_throughput and metrics.get("total_tps"): + dump_metric( + "throughput", + float(metrics["total_tps"]), + labels={"test_case": tc_name, "type": "perf"}, + ) + dump_metric( + "throughput_baseline", + float(self.output_token_throughput), + labels={"test_case": tc_name, "type": "perf"}, + ) + if self.ttft and metrics.get("mean_ttft"): + dump_metric( + "ttft", + float(metrics["mean_ttft"]), + labels={"test_case": tc_name, "type": "perf"}, + ) + dump_metric( + "ttft_baseline", + float(self.ttft), + labels={"test_case": tc_name, "type": "perf"}, + ) + if self.mean_e2e_latency and metrics.get("mean_e2e_latency"): + dump_metric( + "e2e_latency", + float(metrics["mean_e2e_latency"]), + labels={"test_case": tc_name, "type": "perf"}, + ) + dump_metric( + "e2e_latency_baseline", + float(self.mean_e2e_latency), + labels={"test_case": tc_name, "type": "perf"}, + ) + + if self.tpot: + if self.tpot < TPOT_THRESHOLD: + self.assertLessEqual( + float(metrics["mean_tpot"]), + self.tpot + TPOT_TOLERANCE_LOW, + ) + else: + self.assertLessEqual( + float(metrics["mean_tpot"]), + self.tpot * TPOT_TOLERANCE_HIGH, + ) + if self.output_token_throughput: + self.assertGreaterEqual( + float(metrics["total_tps"]), + self.output_token_throughput * OUTPUT_TOKEN_THROUGHPUT_TOLERANCE, + ) + if self.ttft: + self.assertLessEqual( + float(metrics["mean_ttft"]), + self.ttft * TTFT_TOLERANCE, + ) + if self.mean_e2e_latency: + self.assertLessEqual( + float(metrics["mean_e2e_latency"]), + self.mean_e2e_latency * E2E_TOLERANCE, + ) + + +class TestNpuPerformanceTestCaseBase(CustomTestCase): + model = None + benchmark_tool = BENCHMARK_TOOL_DEFAULT + backend = "sglang" + dataset_name = "random" + dataset_path = SHAREGPT_DATASET_TEST_FILE + dataset_type = "gsm8k" # gsm8k | mm-custom-gen + other_args = None + timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + envs = None + max_attempts = 2 + request_rate = None + repeat_rate = None + max_concurrency = None + num_prompts = None + input_len = None + output_len = None + random_range_ratio = 1 + image_resolution = None + image_count = None + warmup_requests = None + seed = None + temperature = None + top_p = None + ttft = None + tpot = None + mean_e2e_latency = None + output_token_throughput = None + + dp = None + generation_kwargs = None + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + env = os.environ.copy() + for key, value in env.items(): + logger.info(f"ENV_VAR_SYS {key}:{value}") + if cls.envs: + for key, value in cls.envs.items(): + logger.info(f"ENV_VAR_CASE {key}:{value}") + env[key] = value + + other_args = list(cls.other_args) + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=cls.timeout, + other_args=other_args, + env=env, + ) + + @classmethod + def tearDownClass(cls): + if hasattr(cls, "process") and cls.process: + try: + kill_process_tree(cls.process.pid) + except Exception as e: + logger.error(f"Error during tearDown: {e}") + + @retry() + def run_throughput(self): + parsed_url = urlparse(self.base_url) + host = parsed_url.hostname + port = parsed_url.port + if self.benchmark_tool == AISBENCHMARK: + metrics = run_aisbench( + host=host, + port=port, + model_path=self.model, + dataset_type=self.dataset_type, + dataset_path=self.dataset_path, + input_len=self.input_len, + output_len=self.output_len, + max_concurrency=self.max_concurrency, + num_prompts=self.num_prompts, + image_resolution=self.image_resolution, + random_range_ratio=self.random_range_ratio, + request_rate=self.request_rate, + repeat_rate=self.repeat_rate, + dp=self.dp, + generation_kwargs=self.generation_kwargs, + ) + assert_metrics(self, metrics) + + else: + bench_params = { + "host": host, + "port": port, + "model_path": self.model, + "backend": self.backend, + "dataset_name": self.dataset_name, + "dataset_path": self.dataset_path, + "request_rate": self.request_rate, + "repeat_rate": self.repeat_rate, + "max_concurrency": self.max_concurrency, + "num_prompts": self.num_prompts, + "input_len": self.input_len, + "output_len": self.output_len, + "random_range_ratio": self.random_range_ratio, + "image_resolution": self.image_resolution, + "image_count": self.image_count, + "warmup_requests": self.warmup_requests, + "seed": self.seed, + "temperature": self.temperature, + "top_p": self.top_p, + } + logger.info(f"Starting benchmark with parameters: {bench_params}") + metrics = run_bench_serving(**bench_params) + assert_metrics(self, metrics) + + +class TestNpuPerfMultiNodePdMixTestCaseBase(CustomTestCase): + model_config = None + benchmark_tool = BENCHMARK_TOOL_DEFAULT + backend = "sglang" + dataset_name = "random" + dataset_path = SHAREGPT_DATASET_TEST_FILE + dataset_type = "gsm8k" # gsm8k | mm-custom-gen + max_attempts = 2 + request_rate = None + repeat_rate = None + max_concurrency = None + num_prompts = None + input_len = None + output_len = None + random_range_ratio = 1 + image_resolution = None + image_count = None + warmup_requests = None + seed = None + temperature = None + top_p = None + ttft = None + tpot = None + mean_e2e_latency = None + output_token_throughput = None + + dp = None + generation_kwargs = None + + @classmethod + def setUpClass(cls): + cls.local_ip = "127.0.0.1" + cls.host = os.getenv("POD_IP") + cls.port = SERVICE_PORT + cls.base_url = f"http://{cls.host}:{cls.port}" + cls.hostname = os.getenv("HOSTNAME") + cls.role = "master" if cls.hostname.endswith("sglang-node-0") else "worker" + logger.info(f"Init {cls.host} {cls.role=}!") + + cls.start_pd_mix_master_node() + cls.start_pd_mix_worker_node() + + @classmethod + def tearDownClass(cls): + pass + + @classmethod + @check_role(allowed_roles=["master"]) + def start_pd_mix_master_node(cls): + sglang_thread = threading.Thread( + target=launch_pd_mix_node, args=(cls.model_config,) + ) + sglang_thread.start() + + wait_server_ready(f"{cls.base_url}/health") + + logger.info( + f"Wait {SERVER_INITIALIZATION_DELAY}s, starting run benchmark ......" + ) + time.sleep(SERVER_INITIALIZATION_DELAY) + + @classmethod + @check_role(allowed_roles=["worker"]) + def start_pd_mix_worker_node(cls): + sglang_thread = threading.Thread( + target=launch_pd_mix_node, args=(cls.model_config,) + ) + sglang_thread.start() + + logger.info( + f"{cls.role} node started, keeping test alive for {MAX_SERVER_KEEP_ALIVE_TIME} seconds" + ) + time.sleep(MAX_SERVER_KEEP_ALIVE_TIME) + + @retry() + @check_role(allowed_roles=["master", "worker"]) + def run_throughput(self): + if self.benchmark_tool == AISBENCHMARK: + metrics = run_aisbench( + host=self.host, + port=str(self.port), + model_path=self.model_config.get("model_path"), + dataset_type=self.dataset_type, + dataset_path=self.dataset_path, + input_len=self.input_len, + output_len=self.output_len, + max_concurrency=self.max_concurrency, + num_prompts=self.num_prompts, + image_resolution=self.image_resolution, + random_range_ratio=self.random_range_ratio, + request_rate=self.request_rate, + repeat_rate=self.repeat_rate, + dp=self.dp, + generation_kwargs=self.generation_kwargs, + ) + assert_metrics(self, metrics) + + else: + bench_params = { + "host": self.host, + "port": str(self.port), + "model_path": self.model_config.get("model_path"), + "backend": self.backend, + "dataset_name": self.dataset_name, + "dataset_path": self.dataset_path, + "request_rate": self.request_rate, + "repeat_rate": self.repeat_rate, + "max_concurrency": self.max_concurrency, + "num_prompts": self.num_prompts, + "input_len": self.input_len, + "output_len": self.output_len, + "random_range_ratio": self.random_range_ratio, + "image_resolution": self.image_resolution, + "image_count": self.image_count, + "warmup_requests": self.warmup_requests, + "seed": self.seed, + "temperature": self.temperature, + "top_p": self.top_p, + } + logger.info(f"Starting benchmark with parameters: {bench_params}") + metrics = run_bench_serving(**bench_params) + assert_metrics(self, metrics) + + +class TestNpuPerfMultiNodePdSepTestCaseBase(CustomTestCase): + model_config = None + benchmark_tool = BENCHMARK_TOOL_DEFAULT + backend = "sglang" + dataset_name = "random" + dataset_path = SHAREGPT_DATASET_TEST_FILE + dataset_type = "gsm8k" # gsm8k | mm-custom-gen + max_attempts = 2 + request_rate = None + repeat_rate = None + max_concurrency = None + num_prompts = None + input_len = None + output_len = None + random_range_ratio = 1 + image_resolution = None + image_count = None + warmup_requests = None + seed = None + temperature = None + top_p = None + ttft = None + tpot = None + mean_e2e_latency = None + output_token_throughput = None + + dp = None + generation_kwargs = None + + @classmethod + def setUpClass(cls): + cls.process = None + cls.local_ip = "127.0.0.1" + cls.host = os.getenv("POD_IP") + cls.port = SERVICE_PORT + cls.base_url = f"http://{cls.host}:{cls.port}" + cls.hostname = os.getenv("HOSTNAME") + cls.role = ( + "router" + if "router" in cls.hostname + else "prefill" if "prefill" in cls.hostname else "decode" + ) + logger.info(f"Init {cls.host} {cls.role=}!") + + cls.start_pd_server() + cls.start_router_server() + + @classmethod + def tearDownClass(cls): + if cls.process: + try: + kill_process_tree(cls.process.pid) + except Exception as e: + logger.error(f"Error during tearDown: {e}") + + @classmethod + @check_role(allowed_roles=["router"]) + def start_router_server(cls): + logger.info(f"Starting router in thread...") + sglang_thread = threading.Thread(target=launch_router, args=(cls.model_config,)) + sglang_thread.daemon = True + sglang_thread.start() + + health_check_url = f"{cls.base_url}/health" + logger.info(f"Waiting for router to be ready at {health_check_url}") + wait_server_ready(health_check_url) + + logger.info( + f"Waiting {SERVER_INITIALIZATION_DELAY} seconds for the server to fully initialize..." + ) + time.sleep(SERVER_INITIALIZATION_DELAY) + + @classmethod + @check_role(allowed_roles=["prefill", "decode"]) + def start_pd_server(cls): + logger.info(f"Starting pd separation node...") + cls.process = launch_pd_separation_node(cls.model_config) + logger.info(f"Pd separation node started with PID: {cls.process.pid}") + + # Loop to check if the process is still running + while True: + if cls.process.poll() is None: + # Process is still running + time.sleep(30) + else: + # Process has exited + exit_code = cls.process.poll() + raise Exception( + f"Sglang process exited on node {cls.host} {cls.hostname} with exit code: {exit_code}" + ) + + @retry() + @check_role(allowed_roles=["router"]) + def run_throughput(self): + if self.benchmark_tool == AISBENCHMARK: + metrics = run_aisbench( + host=self.host, + port=str(self.port), + model_path=self.model_config.get("model_path"), + dataset_type=self.dataset_type, + dataset_path=self.dataset_path, + input_len=self.input_len, + output_len=self.output_len, + max_concurrency=self.max_concurrency, + num_prompts=self.num_prompts, + image_resolution=self.image_resolution, + random_range_ratio=self.random_range_ratio, + request_rate=self.request_rate, + repeat_rate=self.repeat_rate, + dp=self.dp, + generation_kwargs=self.generation_kwargs, + ) + assert_metrics(self, metrics) + + else: + bench_params = { + "host": self.host, + "port": str(self.port), + "model_path": self.model_config.get("model_path"), + "backend": self.backend, + "dataset_name": self.dataset_name, + "dataset_path": self.dataset_path, + "request_rate": self.request_rate, + "repeat_rate": self.repeat_rate, + "max_concurrency": self.max_concurrency, + "num_prompts": self.num_prompts, + "input_len": self.input_len, + "output_len": self.output_len, + "random_range_ratio": self.random_range_ratio, + "image_resolution": self.image_resolution, + "image_count": self.image_count, + "warmup_requests": self.warmup_requests, + "seed": self.seed, + "temperature": self.temperature, + "top_p": self.top_p, + } + logger.info(f"Starting benchmark with parameters: {bench_params}") + metrics = run_bench_serving(**bench_params) + assert_metrics(self, metrics) diff --git a/python/sglang/test/ascend/gsm8k_ascend_mixin.py b/python/sglang/test/ascend/gsm8k_ascend_mixin.py index 44e663acf..95f5713c5 100644 --- a/python/sglang/test/ascend/gsm8k_ascend_mixin.py +++ b/python/sglang/test/ascend/gsm8k_ascend_mixin.py @@ -5,7 +5,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ascend.test_ascend_utils import write_results_to_github_step_summary -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -43,7 +43,6 @@ class GSM8KAscendMixin(ABC): "SGLANG_ENBLE_TORCH_COMILE": "1", "AUTO_USE_UC_MEMORY": "0", "P2P_HCCL_BUFFSIZE": "20", - "TRANSFORMERS_VERBOSITY": os.getenv("TRANSFORMERS_VERBOSITY", "error"), } @classmethod @@ -79,22 +78,23 @@ class GSM8KAscendMixin(ABC): try: args = SimpleNamespace( + max_tokens=512, + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + num_examples=self.num_questions, + num_threads=128, num_shots=self.gsm8k_num_shots, - data_path=None, - num_questions=self.num_questions, - max_new_tokens=512, - parallel=self.gsm8k_parallel, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) metrics = run_eval(args) - model_metrics["accuracy"] = metrics["accuracy"] + model_metrics["accuracy"] = metrics["score"] model_metrics["output_throughput"] = metrics["output_throughput"] model_metrics["latency"] = metrics["latency"] self.assertGreaterEqual( - metrics["accuracy"], + metrics["score"], accuracy_threshold, - f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {accuracy_threshold}', + f'Accuracy of {self.model} is {str(metrics["score"])}, is lower than {accuracy_threshold}', ) self.assertGreaterEqual( metrics["output_throughput"], diff --git a/python/sglang/test/ascend/output_capturer.py b/python/sglang/test/ascend/output_capturer.py new file mode 100644 index 000000000..3d2d8b1aa --- /dev/null +++ b/python/sglang/test/ascend/output_capturer.py @@ -0,0 +1,117 @@ +import os +import select +import threading + + +class OutputCapturer: + """Capture all console print information + + Class Description: + Capture console output using low-level file descriptor redirection. + Used to obtain print information from child processes, NPU processes, + and underlying C/C++ modules that are not logged in sglang logs for test assertion. + All captured output will be displayed normally in the console in real-time. + """ + + def __init__(self): + """Initialize all member variables of the capturer""" + self.old_stdout = None + self.old_stderr = None + self.pipe_out = None + self.pipe_in = None + self.pipe_err_out = None + self.pipe_err_in = None + self.captured_stdout = [] + self.captured_stderr = [] + self.stop_thread = False + self.thread = None + + def start(self): + """Start console output capture""" + # Duplicate and save original stdout/stderr file descriptors + self.old_stdout = os.dup(1) + self.old_stderr = os.dup(2) + + # Create anonymous pipes for output redirection + self.pipe_out, self.pipe_in = os.pipe() + self.pipe_err_out, self.pipe_err_in = os.pipe() + + # Redirect system stdout/stderr to the write end of pipes + os.dup2(self.pipe_in, 1) + os.dup2(self.pipe_err_in, 2) + + # Close unused pipe write ends + os.close(self.pipe_in) + os.close(self.pipe_err_in) + + # Start daemon thread to read output in real time + self.stop_thread = False + self.thread = threading.Thread(target=self._read_loop, daemon=True) + self.thread.start() + + def _read_loop(self): + """The background process reads and prints pipeline data records in a loop.""" + read_fds = [self.pipe_out, self.pipe_err_out] + while not self.stop_thread: + try: + # select listens to multiple file descriptors simultaneously, waiting for data in a non-blocking manner + readable, _, exceptional = select.select(read_fds, [], read_fds, 0.01) + + # Processing file descriptors containing data + for fd in readable: + if fd == self.pipe_out: + data = os.read(fd, 4096) + if data: + self.captured_stdout.append(data) + os.write(self.old_stdout, data) + elif fd == self.pipe_err_out: + err_data = os.read(fd, 4096) + if err_data: + self.captured_stderr.append(err_data) + os.write(self.old_stderr, err_data) + + for fd in exceptional: + if fd in read_fds: + self.stop() + + except OSError: + self.stop() + break + + def get_all(self): + """Get all captured stdout and stderr as UTF-8 string + + Return: Decoded stdout and stderr string (ignore decoding errors) + """ + return self.get_output() + self.get_error() + + def get_output(self): + """Get all captured stdout as UTF-8 string + + Return: Decoded stdout string (ignore decoding errors) + """ + return b"".join(self.captured_stdout).decode("utf-8", errors="ignore") + + def get_error(self): + """Get all captured stderr as UTF-8 string + + Return: Decoded stderr string (ignore decoding errors) + """ + return b"".join(self.captured_stderr).decode("utf-8", errors="ignore") + + def stop(self): + """Stop capture and restore system environment""" + self.stop_thread = True + if self.thread: + self.thread.join(timeout=0.5) + + # Restore original output + os.dup2(self.old_stdout, 1) + os.dup2(self.old_stderr, 2) + + # Close all file descriptors + for fd in [self.pipe_out, self.pipe_err_out, self.old_stdout, self.old_stderr]: + try: + os.close(fd) + except (OSError, IOError): + pass diff --git a/python/sglang/test/ascend/run_eval.py b/python/sglang/test/ascend/run_eval.py new file mode 100644 index 000000000..c56aa7b5f --- /dev/null +++ b/python/sglang/test/ascend/run_eval.py @@ -0,0 +1,122 @@ +import json +import os + +from sglang.test.run_eval import run_eval_once +from sglang.test.simple_eval_common import ( + make_report, + set_ulimit, +) + + +def run_eval(args): + # Lazy import to avoid circular dependency with test_utils + from sglang.test.test_utils import dump_metric + + set_ulimit() + + if "OPENAI_API_KEY" not in os.environ: + os.environ["OPENAI_API_KEY"] = "EMPTY" + + base_url = ( + f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1" + ) + + if args.eval_name == "mmlu": + from sglang.test.ascend.simple_eval_mmlu import MMLUEval + + filename = "https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv" + eval_obj = MMLUEval( + filename, args.num_examples, args.num_threads, getattr(args, "num_shots", 0) + ) + else: + raise ValueError(f"Invalid eval name: {args.eval_name}") + + if getattr(args, "repeat", 1) == 1: + result, latency, sampler = run_eval_once(args, base_url, eval_obj) + metrics = result.metrics | {"score": result.score} + metrics["latency"] = latency + print(f"Total latency: {latency:.3f} s") + print(f"Score: {metrics['score']:.3f}") + + # Compute output throughput from accumulated completion tokens + total_completion_tokens = sum(sampler._completion_tokens) + if total_completion_tokens > 0 and latency > 0: + metrics["output_throughput"] = total_completion_tokens / latency + print(f"Output throughput: {metrics['output_throughput']:.3f} token/s") + + # Report metrics to unified collection framework + dump_metric( + f"{args.eval_name}_score", + metrics["score"], + labels={"model": sampler.model, "eval": args.eval_name}, + ) + dump_metric( + f"{args.eval_name}_latency", + latency, + labels={"model": sampler.model, "eval": args.eval_name}, + ) + else: + from concurrent.futures import ThreadPoolExecutor + + executor = ThreadPoolExecutor(max_workers=args.repeat) + + futures = [ + executor.submit(run_eval_once, args, base_url, eval_obj) + for _ in range(args.repeat) + ] + + scores_repeat = [] + latencies = [] + total_completion_tokens = 0 + + for f in futures: + result, latency, sampler = f.result() + scores_repeat.append(result.score) + latencies.append(latency) + total_completion_tokens += sum(sampler._completion_tokens) + + mean_score = sum(scores_repeat) / len(scores_repeat) + mean_latency = sum(latencies) / len(latencies) + total_latency = sum(latencies) + scores_repeat = [f"{s:.3f}" for s in scores_repeat] + print("=" * 20) + print(f"Repeat: {args.repeat}, mean: {mean_score:.3f}") + print(f"Scores: {scores_repeat}") + print(f"Mean latency: {mean_latency:.3f} s") + print("=" * 20) + metrics = result.metrics | {"scores": scores_repeat} + metrics = metrics | {"mean_score": mean_score} + metrics["latency"] = mean_latency + + if total_completion_tokens > 0 and total_latency > 0: + metrics["output_throughput"] = total_completion_tokens / total_latency + print(f"Output throughput: {metrics['output_throughput']:.3f} token/s") + + # Report metrics to unified collection framework + dump_metric( + f"{args.eval_name}_mean_score", + mean_score, + labels={ + "model": sampler.model, + "eval": args.eval_name, + "repeat": args.repeat, + }, + ) + + executor.shutdown() + + # Dump reports + file_stem = f"{args.eval_name}_{sampler.model.replace('/', '_')}" + report_filename = f"/tmp/{file_stem}.html" + print(f"Writing report to {report_filename}") + with open(report_filename, "w") as fh: + fh.write(make_report(result)) + print(metrics) + result_filename = f"/tmp/{file_stem}.json" + with open(result_filename, "w") as f: + f.write(json.dumps(metrics, indent=2)) + print(f"Writing results to {result_filename}") + + if getattr(args, "return_latency", False): + return metrics, latency + return metrics diff --git a/python/sglang/test/ascend/simple_eval_mmlu.py b/python/sglang/test/ascend/simple_eval_mmlu.py new file mode 100644 index 000000000..588878694 --- /dev/null +++ b/python/sglang/test/ascend/simple_eval_mmlu.py @@ -0,0 +1,125 @@ +# Adapted from https://github.com/openai/simple-evals/ + +""" +Measuring Massive Multitask Language Understanding +Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt +https://arxiv.org/abs/2009.03300 +""" + +import random +import re +from typing import Optional + +import pandas + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + ANSWER_PATTERN_MULTICHOICE, + HTML_JINJA, + Eval, + EvalResult, + SamplerBase, + SingleEvalResult, + format_multichoice_question, +) +from sglang.test.simple_eval_mmlu import subject2category + + +def format_multichoice_question_example(row): + return QUERY_TEMPLATE_MULTICHOICE.format(**row) + + +QUERY_TEMPLATE_MULTICHOICE = """ +Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering. + +{Question} + +A) {A} +B) {B} +C) {C} +D) {D} +""".strip() + +TEMPLATE_MULTICHOICE_EXAMPLE_BEGIN = """ +Answer the multiple-choice questions following the examples below. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. + +""" + +TEMPLATE_MULTICHOICE_EXAMPLE = """ +Example question: +{Question} + +A {A} +B {B} +C {C} +D {D} + +The last line of your response should be +Answer: {Answer} +""".strip() + + +class MMLUEval(Eval): + def __init__( + self, + filename: str, + num_examples: Optional[int], + num_threads: int, + num_shots: int, + ): + if "://" in filename: + df = pandas.read_csv(filename, storage_options={"timeout": 30}) + else: + df = pandas.read_csv(filename) + examples = [row.to_dict() for _, row in df.iterrows()] + if num_shots: + example_questions = "".join( + format_multichoice_question_example(row) + "\n\n" + for row in examples[:num_shots] + ) + self.template = ( + TEMPLATE_MULTICHOICE_EXAMPLE_BEGIN + + example_questions + + QUERY_TEMPLATE_MULTICHOICE + ) + examples = examples[num_shots:] + if num_examples: + examples = random.Random(0).sample(examples, num_examples) + self.examples = examples + self.num_threads = num_threads + self.num_shots = num_shots + + def __call__(self, sampler: SamplerBase) -> EvalResult: + def fn(row: dict): + if self.num_shots: + prompt_messages = [ + sampler._pack_message( + content=self.template.format(**row), role="user" + ) + ] + else: + prompt_messages = [ + sampler._pack_message( + content=format_multichoice_question(row), role="user" + ) + ] + response_text = sampler(prompt_messages) + response_text = response_text or "" + match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text) + extracted_answer = match.group(1) if match else None + score = 1.0 if extracted_answer == row["Answer"] else 0.0 + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=response_text, role="assistant"), + score=score, + correct_answer=row["Answer"], + extracted_answer=extracted_answer, + ) + convo = prompt_messages + [dict(content=response_text, role="assistant")] + category = subject2category.get(row["Subject"], "other") + return SingleEvalResult( + html=html, score=score, metrics={category: score}, convo=convo + ) + + results = common.map_with_progress(fn, self.examples, self.num_threads) + return common.aggregate_results(results) diff --git a/python/sglang/test/ascend/test_ascend_utils.py b/python/sglang/test/ascend/test_ascend_utils.py index af2950591..3e6a9a194 100644 --- a/python/sglang/test/ascend/test_ascend_utils.py +++ b/python/sglang/test/ascend/test_ascend_utils.py @@ -13,13 +13,20 @@ Please remember to sort by variable name within each section. import asyncio import copy +import logging import os +import random import subprocess +import threading +import time from types import SimpleNamespace -from typing import Awaitable, Callable, NamedTuple, Optional +from typing import Awaitable, Callable, List, NamedTuple, Optional + +import requests from sglang.benchmark.serving import run_benchmark from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -29,9 +36,14 @@ from sglang.test.test_utils import ( write_github_step_summary, ) +STDERR_FILENAME = "/tmp/stderr.txt" +STDOUT_FILENAME = "/tmp/stdout.txt" + # Model weights storage directory MODEL_WEIGHTS_DIR = "/root/.cache/modelscope/hub/models/" HF_MODEL_WEIGHTS_DIR = "/root/.cache/huggingface/hub/" +IMAGES_DIR = "/root/.cache/modelscope/hub/datasets/images/" +VIDEO_DIR = "/root/.cache/modelscope/hub/datasets/video/" # LLM model weights path AFM_4_5B_BASE_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "arcee-ai/AFM-4.5B-Base") @@ -46,6 +58,9 @@ CHATGLM2_6B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "ZhipuAI/chatglm2-6b" DBRX_INSTRUCT_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "AI-ModelScope/dbrx-instruct" ) +DEEPSEEK_R1_0528_W8A8_WEIGHTS_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "vllm-ascend/DeepSeek-R1-0528-W8A8" +) DEEPSEEK_V3_2_EXP_W8A8_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "DeepSeek-V3.2-Exp-W8A8" ) @@ -58,6 +73,7 @@ DEEPSEEK_CODER_V2_LITE_WEIGHTS_PATH = os.path.join( DEEPSEEK_CODER_1_3_B_BASE_PATH = os.path.join( MODEL_WEIGHTS_DIR, "deepseek-ai/deepseek-coder-1.3b-base" ) +DOTS_OCR_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "rednote-hilab/dots.ocr") ECO_TECH_QWEN3_32B_W4A4_LAOS_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "Eco-Tech/Qwen3-32B-w4a4-LAOS" ) @@ -75,6 +91,10 @@ GEMMA_4_26B_A4B_IT_WEIGHTS_PATH = os.path.join( ) GEMMA_4_31B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "google/gemma-4-31B-it") GLM_4_9B_CHAT_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "ZhipuAI/glm-4-9b-chat") +GLM_5_1_W4A8_MODEL_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Eco-Tech/GLM-5.1-w4a8") +GPT_OSS_120B_BF16_WEIGHTS_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "eigen-ai-labs/gpt-oss-120b-bf16" +) GRANITE_3_0_3B_A800M_INSTRUCT_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "ibm-granite/granite-3.0-3b-a800m-instruct" ) @@ -82,10 +102,17 @@ GRANITE_3_1_8B_INSTRUCT_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "ibm-granite/granite-3.1-8b-instruct" ) GROK_2_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "huihui-ai/grok-2") +GROK_2_WEIGHTS_TOKENIZER_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "huihui-ai/grok-2/tokenizer.tok.json" +) INTERNLM2_7B_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "Shanghai_AI_Laboratory/internlm2-7b" ) KIMI_K2_THINKING_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Kimi/Kimi-K2-Thinking") +KIMI_K2_5_W4A8_MODEL_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Eco-Tech/Kimi-K2.5-w4a8") +KIMI_K2_5_EAGLE3_MODEL_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "lightseekorg/kimi-k2.5-eagle3" +) LING_LITE_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "inclusionAI/Ling-lite") LLAMA_2_7B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "LLM-Research/Llama-2-7B") LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH = os.path.join( @@ -94,10 +121,20 @@ LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH = os.path.join( LLAMA_3_2_1B_INSTRUCT_TOOL_CALLING_LORA_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "codelion/Llama-3.2-1B-Instruct-tool-calling-lora" ) +LLAMA_3_2_1B_INSTRUCT_TOOL_FAST_LORA_WEIGHTS_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "suayptalha/FastLlama-3.2-LoRA" +) LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "LLM-Research/Llama-3.2-1B-Instruct" ) LLAMA_3_2_1B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "LLM-Research/Llama-3.2-1B") +LLAMA_3_8B_EAGLE_WEIGHTS_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "lmsys/sglang-EAGLE-LLaMA3-Instruct-8B" +) +LLAMA_3_8B_INSTRUCT_WEIGHTS_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "LLM-Research/Meta-Llama-3-8B-Instruct" +) + LLAMA_4_SCOUT_17B_16E_INSTRUCT_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "meta-llama/Llama-4-Scout-17B-16E-Instruct" ) @@ -113,6 +150,9 @@ MINICPM3_4B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "OpenBMB/MiniCPM3-4B" MISTRAL_7B_INSTRUCT_V0_2_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "mistralai/Mistral-7B-Instruct-v0.2" ) +OLMO_2_1124_7B_INSTRUCT_WEIGHTS_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "allenai/OLMo-2-1124-7B-Instruct" +) OLMOE_1B_7B_0924_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "allenai/OLMoE-1B-7B-0924" ) @@ -126,12 +166,16 @@ QWEN2_5_7B_INSTRUCT_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "Qwen/Qwen2.5-7B-Instruct" ) QWEN3_0_6B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Qwen/Qwen3-0.6B") +QWEN3_5_27B_MODEL_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Qwen/Qwen3.5-27B") QWEN3_1_7B_GPTQ_INT8_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "Qwen/Qwen3-1.7B-GPTQ-Int8" ) QWEN3_235B_A22B_W8A8_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "vllm-ascend/Qwen3-235B-A22B-W8A8" ) +QWEN3_235B_A22B_EAGLE_MODEL_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "Qwen/Qwen3-235B-A22B-Eagle3" +) QWEN3_30B_A3B_GPTQ_2507_INT4_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "Qwen/Qwen3-30B-A3B-GPTQ-Int4" ) @@ -152,6 +196,10 @@ QWEN3_8B_INT4_AUTOROUND_WEIGHTS_PATH = os.path.join( ) QWEN3_8B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Qwen/Qwen3-8B") QWEN3_8B_EAGLE3_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Qwen/Qwen3-8B_eagle3") +QWEN3_8B_DECRYPTED_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "YZY/Qwen3-8B") +QWEN3_8B_EAGLE3_DECRYPTED_WEIGHTS_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "YZY/Qwen3-8B_eagle3" +) QWEN3_32B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Qwen/Qwen3-32B") QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "Qwen3-Coder-480B-A35B-Instruct-w8a8-QuaRot" @@ -159,18 +207,33 @@ QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH = os.path.join( QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "Qwen/Qwen3-Next-80B-A3B-Instruct" ) -QWEN3_32B_EAGLE3_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Qwen/Qwen3-32B-Eagle3") +QWEN3_32B_EAGLE3_WEIGHTS_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "Zjcxy-SmartAI/Qwen3-32B-Eagle3" +) QWEN3_32B_W8A8_MINDIE_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "aleoyang/Qwen3-32B-w8a8-MindIE" ) QWQ_32B_W8A8_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "vllm-ascend/QWQ-32B-W8A8") SMOLLM_1_7B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "HuggingFaceTB/SmolLM-1.7B") +SOLAR_10_7B_INSTRUCT_V1_0_WEIGHTS_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "upstage/SOLAR-10.7B-Instruct-v1.0" +) STABLELM_2_1_6B_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "stabilityai/stablelm-2-1_6b" ) -XVERSE_MOE_A36B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "xverse/XVERSE-MoE-A36B") +STARCODER2_7B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "bigcode/starcoder2-7b") TRINITY_MINI_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "arcee-ai/Trinity-Mini") +XVERSE_MOE_A36B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "xverse/XVERSE-MoE-A36B") MINIMAX_M2_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "cyankiwi/MiniMax-M2-BF16") +MINIMAX_M2_5_W8A8_MODEL_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "Eco-Tech/MiniMax-M2.5-w8a8-QuaRot" +) +MINIMAX_M2_5_EAGLE3_MODEL_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "sgl-npu/MiniMax-M2.5-eagel-model-0318" +) +EAGLE3_LLAMA3_1_INSTRUCT_8B_WEIGHTS_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "sglang-EAGLE3-LLaMA3.1-Instruct-8B" +) # VLM model weights path DEEPSEEK_VL2_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "deepseek-ai/deepseek-vl2") @@ -225,14 +288,28 @@ QWEN3_30B_A3B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Qwen/Qwen3-30B-A3B QWEN3_30B_A3B_W8A8_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "Qwen/Qwen3-30B-A3B-w8a8" ) +DEEPSEEK_V2_LITE_W8A8_WEIGHTS_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "vllm-ascend/DeepSeek-V2-Lite-W8A8" +) DEEPSEEK_R1_DISTILL_QWEN_7B_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B" ) - +DEEPSEEK_R1_0528_W4A8_PER_CHANNEL_WEIGHTS_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "DeepSeek-R1-0528-w4a8-per-channel" +) +DEEPSEEK_R1_0528_W8A8_WEIGHTS_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "vllm-ascend/DeepSeek-R1-0528-W8A8" +) QWEN3_30B_MODELSLIM_INT4_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "Eco-Tech/Qwen3-30B-A3B-w4a4-LAOS" ) +QWEN3_5_397B_W4A8_MODEL_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "Eco-Tech/Qwen3.5-397B-A17B-w4a8-mtp" +) +QWEN3_5_397B_W8A8_MODEL_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp" +) # Embedding model weights path BGE_LARGE_EN_V1_5_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "bge-large-en-v1.5") @@ -274,10 +351,37 @@ SKYWORK_REWARD_LLAMA_3_1_8B_V0_2_WEIGHTS_PATH = os.path.join( HF_MODEL_WEIGHTS_DIR, "models--Skywork--Skywork-Reward-Llama-3.1-8B-v0.2/snapshots/d4117fbfd81b72f41b96341238baa1e3e90a4ce1", ) +KIMI_K2_6_W4A8_MODEL_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Eco-Tech/Kimi-K2.6-w4a8") +KIMI_K2_6_EAGLE3_MODEL_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "lightseekorg/kimi-k2.6-eagle3" +) +GLM_4_6V_FLASH_MODEL_PATH = os.path.join(MODEL_WEIGHTS_DIR, "ZhipuAI/GLM-4.6V-Flash") +QWEN3_VL_8B_THINKING_MODEL_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "Qwen/Qwen3-VL-8B-Thinking" +) +QWEN3_VL_30B_A3B_THINKING_MODEL_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "Qwen/Qwen3-VL-30B-A3B-Thinking" +) +QWEN3_OMNI_30B_A3B_THINKING_MODEL_PATH = os.path.join( + MODEL_WEIGHTS_DIR, "Qwen/Qwen3-Omni-30B-A3B-Thinking" +) +# Images path +IMAGES_EXAMPLE_PATH = os.path.join(IMAGES_DIR, "example_image.png") +IMAGES_023_PATH = os.path.join(IMAGES_DIR, "023.jpg") +IMAGES_MAN_PATH = os.path.join(IMAGES_DIR, "man.png") +IMAGES_LOGO_PATH = os.path.join(IMAGES_DIR, "logo.png") +VIDEO_JOBS_PATH = os.path.join(VIDEO_DIR, "jobs.mp4") +INVOICE_WITH_BARCODE_LOGO_IMAGES_PATH = os.path.join( + IMAGES_DIR, "invoice_with_barcode_logo.jpeg" +) # fmt: on # Other DEEPSEEK_CODER_JSON_PATH = "/__w/sglang/sglang/test/registered/ascend/basic_function/parameter/deepseek_coder.json" +FR_SPEC_TOKEN_MAP_PATH = "/root/.cache/sglang/FR-Spec/freq_32768.pt" +CONFIG_YAML_PATH = ( + "/__w/sglang/sglang/test/registered/ascend/basic_function/config/config.yaml" +) class ModelTestConfig(NamedTuple): @@ -371,40 +475,6 @@ def get_benchmark_args( header=None, max_concurrency=None, ): - """Constructing the parameter objects needed for inference tests - - Parameters: - base_url: url - backend: Inference backend - dataset_name: Data set name - dataset_path: Dataset path - tokenizer: tokenizer - num_prompts: Total number of test requests - sharegpt_output_len: Output the number of tokens - random_input_len: The length of the randomly generated input prompt - random_output_len: The length of the randomly generated output prompt - sharegpt_context_len: Sharegpt dataset context length - request_rate: Request rate - disable_stream: Disable streaming output - disable_ignore_eos: Should eos_token be ignored? - seed: random seed - device: Device type - pd_separated: Enable PD separation - lora_name: LoRA fine-tuning model path - lora_request_distribution: LoRA request distribution strategy - lora_zipf_alpha: Control request distribution skewness - gsp_num_groups: Grouped Sequence Parallelism - gsp_prompts_per_group: Number of parallel prompts within each group - gsp_system_prompt_len: GSP system prompts length - gsp_question_len: GSP question length - gsp_output_len: GSP output length - gsp_num_turns: GSP Dialogue Rounds - header: HTTP request header - max_concurrency: Maximum number of concurrent requests - Returns: - The return parameter is the same as the input. - """ - return SimpleNamespace( backend=backend, base_url=base_url, @@ -474,6 +544,7 @@ def run_bench_serving( max_concurrency=None, background_task: Optional[Callable[[str, asyncio.Event], Awaitable[None]]] = None, lora_name: Optional[str] = None, + timeout_for_server_launch=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, ): """Start the service and obtain the inference results. @@ -501,6 +572,7 @@ def run_bench_serving( max_concurrency: Maximum number of concurrent requests background_task: Background tasks lora_name: LoRA fine-tuning model path + timeout_for_server_launch: Raise the service timeout period Returns: res: Number of requests successfully completed @@ -513,7 +585,7 @@ def run_bench_serving( process = popen_launch_server( model, base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + timeout=timeout_for_server_launch, other_args=other_server_args, ) @@ -574,7 +646,279 @@ def run_bench_serving( return res +# hook factory +def create_attention_monitor_hook_factory(config): + """ + Factory function to create a forward hook for monitoring self-attention layer states. + This hook records input/output statistics during model forward propagation. + + Args: + config (dict): Configuration dictionary containing hook parameters + layer_index (int): Index of the target attention layer to monitor + + Returns: + function: Forward hook function to be registered on the target module + """ + # Get target layer index from config, default to 0 if not specified + layer_index = config.get("layer_index", 0) + + # Initialize logging configuration if no handlers are set + if not logging.root.handlers: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + def attention_monitor_hook(module, inputs, output): + """ + Forward hook function that monitors and logs the internal states of a self-attention layer. + Executed automatically during the forward pass of the module it is registered to. + + Args: + module (torch.nn.Module): The module this hook is attached to + inputs (tuple): Input tensors passed to the module's forward method + output (torch.Tensor): Output tensor returned by the module's forward method + + Returns: + torch.Tensor: Unmodified output tensor to preserve model computation flow + """ + # Record current timestamp for time-series tracking + timestamp = time.time() + + # Extract hidden states from inputs (second input tensor of attention layer) + hidden_states = inputs[1] if inputs and len(inputs) > 1 else None + + # Construct monitoring record with key statistics + monitor_record = { + "timestamp": timestamp, + "layer_index": layer_index, + "module_type": type(module).__name__, + # Compute sum of hidden states across last dim, take first 5 elements for logging + "inputs": hidden_states.sum(-1)[:5] if hidden_states is not None else None, + # Compute sum of output across last dim, take first 5 elements for logging + "outputs": output.sum(-1)[:5], + } + + # Log the monitoring record + logging.info(f"hook effect: {monitor_record}") + + # Return the original output to maintain normal model forward propagation + return output + + return attention_monitor_hook + + +def read_output(output_lines: List[str], filename: str = STDERR_FILENAME): + """Print the output in real time with another thread.""" + while not os.path.exists(filename): + time.sleep(0.01) + + pt = 0 + while pt >= 0: + if pt > 0 and not os.path.exists(filename): + break + try: + lines = open(filename).readlines() + except FileNotFoundError: + print(f"{pt=}, {os.path.exists(filename)=}") + raise + for line in lines[pt:]: + print(line, end="", flush=True) + output_lines.append(line) + pt += 1 + time.sleep(0.1) + + +def run_and_check_memory_leak( + workload_func, + disable_radix_cache, + enable_mixed_chunk, + disable_overlap, + chunked_prefill_size, + assert_has_abort, + api_key: Optional[str] = None, +): + other_args = [ + "--chunked-prefill-size", + str(chunked_prefill_size), + "--log-level", + "debug", + ] + if disable_radix_cache: + other_args += ["--disable-radix-cache"] + if enable_mixed_chunk: + other_args += ["--enable-mixed-chunk"] + if disable_overlap: + other_args += ["--disable-overlap-schedule"] + + model = LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH + port = random.randint(4000, 5000) + base_url = f"http://127.0.0.1:{port}" + + # Create files and launch the server + stdout = open(STDOUT_FILENAME, "w") + stderr = open(STDERR_FILENAME, "w") + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + return_stdout_stderr=(stdout, stderr), + api_key=api_key, + ) + + # Launch a thread to stream the output + output_lines = [] + t = threading.Thread(target=read_output, args=(output_lines,)) + t.start() + + # Run the workload + workload_func(base_url, model) + + # Clean up everything + kill_process_tree(process.pid) + stdout.close() + stderr.close() + if os.path.exists(STDOUT_FILENAME): + os.remove(STDOUT_FILENAME) + if os.path.exists(STDERR_FILENAME): + os.remove(STDERR_FILENAME) + kill_process_tree(process.pid) + t.join() + + # Assert success + has_new_server = False + has_leak = False + has_abort = False + for line in output_lines: + if "Uvicorn running" in line: + has_new_server = True + if "leak" in line: + has_leak = True + if "Abort" in line: + has_abort = True + + assert has_new_server + assert not has_leak + if assert_has_abort: + assert has_abort + + +def run_mmlu_test( + disable_radix_cache=False, + enable_mixed_chunk=False, + disable_overlap=False, + chunked_prefill_size=32, +): + def workload_func(base_url, model): + # Run the eval + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=128, + num_threads=128, + ) + + try: + metrics = run_eval(args) + assert metrics["score"] >= 0.65, f"{metrics=}" + finally: + pass + + run_and_check_memory_leak( + workload_func, + disable_radix_cache, + enable_mixed_chunk, + disable_overlap, + chunked_prefill_size, + assert_has_abort=False, + ) + + +def send_concurrent_requests( + base_url: str, + num_requests: int, + num_concurrent: int = 8, + input_text: str = "The capital of France is", + max_new_tokens: int = 32, + temperature: float = 0.0, + request_timeout: int = 60, +) -> list: + """Send multiple concurrent HTTP POST requests to the /generate endpoint. + + Uses threading (NOT asyncio + blocking calls) to achieve true concurrency. + asyncio.gather() combined with synchronous requests.post() does not produce + real parallelism; threading is required for concurrent blocking I/O. + + Parameters: + base_url: Server base URL, e.g. "http://127.0.0.1:30000" + num_requests: Total number of requests to send + num_concurrent: Maximum in-flight requests at any given time (semaphore) + input_text: Text prompt sent to every request + max_new_tokens: Maximum new tokens to generate per request + temperature: Sampling temperature (0 = greedy / deterministic) + request_timeout: Per-request HTTP timeout in seconds; raises on exceed + + Returns: + Unsorted list of result dicts, one per request, each with: + task_id (int) -- zero-based request index + status_code (int)-- HTTP status code, or -1 on exception + text (str) -- response body, or exception message on failure + """ + + results: list = [] + lock = threading.Lock() + semaphore = threading.Semaphore(num_concurrent) + + def _send_one(task_id: int) -> None: + semaphore.acquire() + try: + response = requests.post( + f"{base_url}/generate", + json={ + "text": input_text, + "sampling_params": { + "temperature": temperature, + "max_new_tokens": max_new_tokens, + }, + }, + timeout=request_timeout, + ) + with lock: + results.append( + { + "task_id": task_id, + "status_code": response.status_code, + "text": response.text, + } + ) + except Exception as exc: + with lock: + results.append( + { + "task_id": task_id, + "status_code": -1, + "text": str(exc), + } + ) + finally: + semaphore.release() + + threads = [ + threading.Thread(target=_send_one, args=(i,)) for i in range(num_requests) + ] + for t in threads: + t.start() + for t in threads: + t.join() + + return results + + HEADER = """ +### Models | Model | Server | Client | Output Throughput | Expected Output Throughput | Latency | Expected Latency | Accuracy | Expected Accuracy | Status | | ----- | ------ | ------ | -------- | ------------------ | ------- | ---------------- | -------- | --------- | ------ | """ diff --git a/python/sglang/test/ascend/test_embedding_base.py b/python/sglang/test/ascend/test_embedding_base.py new file mode 100644 index 000000000..2857b8317 --- /dev/null +++ b/python/sglang/test/ascend/test_embedding_base.py @@ -0,0 +1,102 @@ +import multiprocessing as mp +from abc import ABC +from typing import List, Optional, Tuple + +import torch +from transformers import AutoConfig, AutoTokenizer + +from sglang.test.runners import DEFAULT_PROMPTS, HFRunner, SRTRunner +from sglang.test.test_utils import get_similarities + + +class BaseEmbeddingTest(ABC): + """Base test class for embedding model tests""" + + MODELS: List[ + Tuple[str, int, float] + ] # [(model_path, tp_size, prefill_tolerance), ...] + TORCH_DTYPES: List[torch.dtype] = [torch.float16] + DEFAULT_PROMPTS: List[str] = DEFAULT_PROMPTS + DEFAULT_MAX_LENGTH: int = 2048 + + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def _truncate_prompts(self, prompts, model_path): + """Truncate prompts to model's max length""" + config = AutoConfig.from_pretrained(model_path) + max_length = getattr(config, "max_position_embeddings", self.DEFAULT_MAX_LENGTH) + + tokenizer = AutoTokenizer.from_pretrained(model_path) + + truncated_prompts = [] + for prompt in prompts: + tokens = tokenizer(prompt, return_tensors="pt", truncation=False) + if len(tokens.input_ids[0]) > max_length: + truncated_text = tokenizer.decode( + tokens.input_ids[0][: max_length - 1], skip_special_tokens=True + ) + truncated_prompts.append(truncated_text) + else: + truncated_prompts.append(prompt) + return truncated_prompts + + def assert_close_prefill_logits( + self, + prompts, + model_path, + tp_size, + torch_dtype, + prefill_tolerance, + matryoshka_dim: Optional[int] = None, + ) -> None: + """Assert embeddings from HF and SRT are within tolerance""" + truncated_prompts = self._truncate_prompts(prompts, model_path) + + with HFRunner( + model_path, + torch_dtype=torch_dtype, + model_type="embedding", + matryoshka_dim=matryoshka_dim, + ) as hf_runner: + hf_outputs = hf_runner.forward(truncated_prompts) + + attention_backend = "ascend" + json_model_override_args = ( + {"matryoshka_dimensions": [matryoshka_dim]} if matryoshka_dim else None + ) + + with SRTRunner( + model_path, + tp_size=tp_size, + torch_dtype=torch_dtype, + model_type="embedding", + attention_backend=attention_backend, + json_model_override_args=json_model_override_args, + ) as srt_runner: + srt_outputs = srt_runner.forward( + truncated_prompts, dimensions=matryoshka_dim + ) + + for i in range(len(prompts)): + hf_logits = torch.Tensor(hf_outputs.embed_logits[i]) + srt_logits = torch.Tensor(srt_outputs.embed_logits[i]) + + similarity = torch.tensor(get_similarities(hf_logits, srt_logits)) + print("similarity diff", abs(similarity - 1)) + + if len(prompts[i]) <= 1000: + assert torch.all( + abs(similarity - 1) < prefill_tolerance + ), "embeddings are not all close" + + def test_prefill_logits(self): + """Main test method to run for all models and dtypes""" + models_to_test = self.MODELS + + for model, tp_size, prefill_tolerance in models_to_test: + for torch_dtype in self.TORCH_DTYPES: + self.assert_close_prefill_logits( + self.DEFAULT_PROMPTS, model, tp_size, torch_dtype, prefill_tolerance + ) diff --git a/python/sglang/test/ascend/test_no_hf_reward_base.py b/python/sglang/test/ascend/test_no_hf_reward_base.py new file mode 100644 index 000000000..94fb16253 --- /dev/null +++ b/python/sglang/test/ascend/test_no_hf_reward_base.py @@ -0,0 +1,59 @@ +import multiprocessing as mp +from abc import ABC + +import torch + +from sglang.test.runners import SRTRunner + +PROMPT = ( + "What is the range of the numeric output of a sigmoid node in a neural network?" +) +RESPONSE1 = "The output of a sigmoid node is bounded between -1 and 1." +RESPONSE2 = "The output of a sigmoid node is bounded between 0 and 1." + +CONVS = [ + [{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE1}], + [{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE2}], +] + + +class BaseNoHFRewardModelTest(ABC): + """Base test class for reward model testing that doesn't compare with HF. + + This is for models that only need to verify SGLang can run them successfully. + """ + + # Required attributes for subclasses + model_path: str + + # Optional attributes with defaults + torch_dtype: torch.dtype = torch.float16 + tp_size: int = 4 + trust_remote_code: bool = True + disable_cuda_graph: bool = True + mem_fraction_static: float = 0.8 + + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def test_assert_close_reward_scores(self): + """Test that the model can generate reward scores.""" + srt_runner_kwargs = { + "trust_remote_code": self.trust_remote_code, + "disable_cuda_graph": self.disable_cuda_graph, + "tp_size": self.tp_size, + "mem_fraction_static": self.mem_fraction_static, + } + + with SRTRunner( + self.model_path, + torch_dtype=self.torch_dtype, + model_type="reward", + **srt_runner_kwargs, + ) as srt_runner: + prompts = srt_runner.tokenizer.apply_chat_template(CONVS, tokenize=False) + srt_outputs = srt_runner.forward(prompts) + srt_scores = torch.tensor(srt_outputs.scores) + print(f"accuracy: {srt_scores}") + self.assertIsInstance(srt_scores, torch.Tensor) diff --git a/python/sglang/test/ascend/test_npu_logging.py b/python/sglang/test/ascend/test_npu_logging.py new file mode 100644 index 000000000..6ef799f96 --- /dev/null +++ b/python/sglang/test/ascend/test_npu_logging.py @@ -0,0 +1,153 @@ +import os +import re +import tempfile +import time + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.test_ascend_utils import LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestNPULoggingBase(CustomTestCase): + """Testcase：Test base class to verify whether the parameters in the logging function are correct. + + Description: + Includes methods for initializing data and methods for verifying the correctness of the logging function. + + [Test Category] Parameter + [Test Target] --log-requests; --log-requests-level; --log-requests-target; --uvicorn-access-log-exclude-prefixes; + --enable-metrics; --enable-metrics-for-all-scheduler; + --bucket-time-to-first-token; --bucket-inter-token-latency; --bucket-e2e-request-latency; + --collect-tokens-histogram; --prompt-tokens-buckets; --generation-tokens-buckets; + --tokenizer-metrics-custom-labels-header; --tokenizer-metrics-allowed-custom-labels; + --gc-warning-threshold-secs + """ + + @staticmethod + def get_lines_with_keyword(filename, keyword): + """Find and return lines matching a regex keyword from a specified file, with line numbers and content. + + Function Description: + Reads the target file line by line, uses the input keyword as a regular expression pattern to match each line's content. + For each line that matches the regex pattern, encapsulates the line number (1-indexed) and content into a dictionary, + and finally returns a list of dictionaries containing all matched lines. + + Args: + filename (str): Path to the file to be read + keyword (str): Regular expression pattern for matching + + Returns: + List[Dict[str, Union[str, int]]] + List of dictionaries for matched lines, each dictionary contains two key-value pairs: + - "line_number": int - Line number of the matched line (starts from 1) + - "content": str - Full text content of the matched line + """ + results = [] + try: + with open(filename, "r", encoding="utf-8") as file: + for line_num, line in enumerate(file, 1): + if re.match(keyword, line): + results.append( + { + "line_number": line_num, + "content": line.strip(), + } + ) + return results + except Exception as e: + print(f"error:{e}") + return [] + + @classmethod + def setUpClass(cls): + cls.model = LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.other_args = [ + "--trust-remote-code", + "--mem-fraction-static", + "0.8", + "--attention-backend", + "ascend", + "--disable-cuda-graph", + "--log-requests", + ] + cls.out_log_file_obj = tempfile.NamedTemporaryFile( + mode="w+", encoding="utf-8", delete=False, suffix=".txt" + ) + cls.out_log_name = cls.out_log_file_obj.name + cls.out_log_file = cls.out_log_file_obj + cls.err_log_file_obj = tempfile.NamedTemporaryFile( + mode="w+", encoding="utf-8", delete=False, suffix=".txt" + ) + cls.err_log_name = cls.err_log_file_obj.name + cls.err_log_file = cls.err_log_file_obj + cls.process = None + + @classmethod + def tearDownClass(cls): + if cls.process: + kill_process_tree(cls.process.pid) + cls.out_log_file.close() + os.remove(cls.out_log_name) + cls.err_log_file.close() + os.remove(cls.err_log_name) + + @classmethod + def launch_server(cls): + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=cls.other_args, + return_stdout_stderr=(cls.out_log_file, cls.err_log_file), + ) + + def inference_once(self, max_tokens=32): + response = requests.post( + f"{self.base_url}/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_tokens, + }, + }, + ) + + self.assertEqual(response.status_code, 200, "Failed to call generate API") + self.assertIn("Paris", response.text, "Inference out error.") + + def wait_for_log_content(self, timeout=30): + """Wait for and return the content of the specified log file, with timeout handling. + + Function Description: + Continuously reads the target log file in a loop within the set timeout period, + avoids assertion failures caused by reading the log too early before log writing is completed. + Returns the log content immediately once the file has non-empty content, + otherwise waits and retries reading at intervals until the timeout is reached. + + Args: + timeout (int, optional): Maximum waiting time in seconds, default value is 30 seconds. + + Returns: + str + Full text content read from the log file: + - Non-empty string if log content is detected within the timeout period + - Empty string if no log content is found after the timeout expires + """ + start_time = time.time() + content = "" + while time.time() - start_time < timeout: + with open(self.out_log_file.name, "r", encoding="utf-8") as f: + content = f.read() + if content: + break + time.sleep(0.5) + return content diff --git a/python/sglang/test/ascend/vlm_utils.py b/python/sglang/test/ascend/vlm_utils.py index ca9cfe8fe..cab91eeb8 100644 --- a/python/sglang/test/ascend/vlm_utils.py +++ b/python/sglang/test/ascend/vlm_utils.py @@ -1,10 +1,9 @@ -import glob -import json import os -import subprocess +import warnings +from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.ascend.test_ascend_utils import write_results_to_github_step_summary +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -43,195 +42,56 @@ class TestVLMModels(CustomTestCase): os.environ["OPENAI_API_KEY"] = cls.api_key os.environ["OPENAI_API_BASE"] = f"{cls.base_url}/v1" - os.environ["TRANSFORMERS_VERBOSITY"] = os.getenv( - "TRANSFORMERS_VERBOSITY", "error" + # Prepare environment variables + process_env = os.environ.copy() + + cls.process = popen_launch_server( + cls.model, + base_url=cls.base_url, + timeout=cls.timeout_for_server_launch, + api_key=cls.api_key, + other_args=cls.other_args, + env=process_env, ) - def run_mmmu_eval( - self, - model_version: str, - output_path: str, - limit: str, - *, - env: dict | None = None, - ): - """ - Evaluate a VLM on the MMMU validation set with lmms‑eval. - Only `model_version` (checkpoint) and `chat_template` vary; - We are focusing only on the validation set due to resource constraints. - """ - # -------- fixed settings -------- - model = "openai_compatible" - tp = 1 - tasks = "mmmu_val" - batch_size = 2 - log_suffix = "openai_compatible" - os.makedirs(output_path, exist_ok=True) - - # -------- compose --model_args -------- - model_args = f'model_version="{model_version}",' f"tp={tp}" - - # -------- build command list -------- - cmd = [ - "python3", - "-m", - "lmms_eval", - "--model", - model, - "--model_args", - model_args, - "--tasks", - tasks, - "--batch_size", - str(batch_size), - "--log_samples", - "--log_samples_suffix", - log_suffix, - "--output_path", - str(output_path), - "--limit", - limit, - "--config", - "/__w/sglang/sglang/test/registered/ascend/vlm_models/mmmu-val.yaml", - ] - - subprocess.run( - cmd, - check=True, - timeout=3600, - ) - - return subprocess.list2cmdline(cmd) # Return the command for logging purposes - - def _run_vlm_mmmu_test( - self, - output_path="./logs", - test_name="", - custom_env=None, - capture_output=False, - limit="50", - ): - """ - Common method to run VLM MMMU benchmark test. - Args: - model: Model to test - output_path: Path for output logs - test_name: Optional test name for logging - custom_env: Optional custom environment variables - capture_output: Whether to capture server stdout/stderr - """ - print(f"\nTesting model: {self.model}{test_name}") - - model_metrics = { - "server": subprocess.list2cmdline(map(str, self.other_args)), - "client": "mmmu_eval", - "accuracy_threshold": self.mmmu_accuracy, - } - - process = None - server_output = "" - mmmu_accuracy = None - - try: - # Prepare environment variables - process_env = os.environ.copy() - if custom_env: - process_env.update(custom_env) - - # Prepare stdout/stderr redirection if needed - stdout_file = None - stderr_file = None - if capture_output: - stdout_file = open("/tmp/server_stdout.log", "w") - stderr_file = open("/tmp/server_stderr.log", "w") - - process = popen_launch_server( - self.model, - base_url=self.base_url, - timeout=self.timeout_for_server_launch, - api_key=self.api_key, - other_args=self.other_args, - env=process_env, - return_stdout_stderr=( - (stdout_file, stderr_file) if capture_output else None - ), - ) - - model_metrics["server"] = subprocess.list2cmdline(process.args) - - # Run evaluation - model_metrics["client"] = self.run_mmmu_eval(self.model, output_path, limit) - - # Get the result file - result_file_path = glob.glob(f"{output_path}/*.json")[0] - - with open(result_file_path, "r") as f: - result = json.load(f) - print(f"Result{test_name}\n: {result}") - - # Process the result - mmmu_accuracy = result["results"]["mmmu_val"]["mmmu_acc,none"] - print( - f"Model {self.model} achieved accuracy{test_name}: {mmmu_accuracy:.4f}" - ) - - # Capture server output if requested - if capture_output and process: - server_output = self._read_output_from_files() - - model_metrics["accuracy"] = mmmu_accuracy - - # Assert performance meets expected threshold - self.assertGreaterEqual( - mmmu_accuracy, - self.mmmu_accuracy, - f"Model {self.model} accuracy ({mmmu_accuracy:.4f}) below expected threshold ({self.mmmu_accuracy:.4f}){test_name}", - ) - - return server_output - - except Exception as e: - model_metrics["error"] = e - print(f"Error testing {self.model}{test_name}: {e}") - self.fail(f"Test failed for {self.model}{test_name}: {e}") - finally: - write_results_to_github_step_summary({self.model: model_metrics}) - - # Ensure process cleanup happens regardless of success/failure - if process is not None and process.poll() is None: - print(f"Cleaning up process {process.pid}") - try: - kill_process_tree(process.pid) - except Exception as e: - print(f"Error killing process: {e}") - - # clean up temporary files - if capture_output: - if stdout_file: - stdout_file.close() - if stderr_file: - stderr_file.close() - for filename in ["/tmp/server_stdout.log", "/tmp/server_stderr.log"]: - try: - if os.path.exists(filename): - os.remove(filename) - except Exception as e: - print(f"Error removing {filename}: {e}") - - def _read_output_from_files(self): - output_lines = [] - - log_files = [ - ("/tmp/server_stdout.log", "[STDOUT]"), - ("/tmp/server_stderr.log", "[STDERR]"), - ] - for filename, tag in log_files: + @classmethod + def tearDownClass(cls): + if cls.process and cls.process.poll() is None: + print(f"Cleaning up server process {cls.process.pid}") try: - if os.path.exists(filename): - with open(filename, "r") as f: - for line in f: - output_lines.append(f"{tag} {line.rstrip()}") + kill_process_tree(cls.process.pid) except Exception as e: - print(f"Error reading {tag.lower()} file: {e}") + print(f"Error killing server process: {e}") - return "\n".join(output_lines) + def _run_vlm_mmmu_test(self, test_name=""): + warnings.filterwarnings( + "ignore", category=ResourceWarning, message="unclosed.*socket" + ) + + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmmu", + num_examples=100, + num_threads=64, + max_tokens=30, + return_latency=True, + ) + + metrics, latency = run_eval(args) + + metrics["score"] = round(metrics["score"], 4) + metrics["latency"] = round(latency, 4) + + print( + f"\n{'=' * 42}\n" + f"{self.model} - metrics={metrics} score={metrics['score']}\n" + f"{'=' * 42}\n" + ) + + self.assertGreaterEqual( + metrics["score"], + self.mmmu_accuracy, + f"Model {self.model} accuracy ({metrics['score']}) " + f"below expected threshold ({self.mmmu_accuracy:.4f}){test_name}", + ) diff --git a/test/registered/ascend/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py b/test/registered/ascend/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py new file mode 100644 index 000000000..f5d4756b6 --- /dev/null +++ b/test/registered/ascend/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py @@ -0,0 +1,46 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.test_ascend_utils import DEEPSEEK_V3_2_EXP_W8A8_WEIGHTS_PATH +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="accuracy testcase", +) + +OTHER_ARGS = [ + "--trust-remote-code", + "--mem-fraction-static", + "0.9", + "--attention-backend", + "ascend", + "--disable-cuda-graph", + "--tp-size", + "16", + "--quantization", + "modelslim", + "--disable-radix-cache", +] + + +class TestNPUDeepSeek_V3_2_8P_AIME2025(TestNpuAccuracyTestCaseBase): + + model = DEEPSEEK_V3_2_EXP_W8A8_WEIGHTS_PATH + other_args = OTHER_ARGS + accuracy = 0.931 + datasets = ["aime25"] + few_shot_num = 0 + generation_config = {"max_tokens": 65536, "temperature": 1.0} + eval_batch_size = 64 + + def test_aime2025(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py b/test/registered/ascend/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py new file mode 100644 index 000000000..c51f898fb --- /dev/null +++ b/test/registered/ascend/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py @@ -0,0 +1,71 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_6V_FLASH_MODEL_PATH +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_BUFFSIZE": "1000", + "HCCL_OP_EXPANSION_MODE": "AIV", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "SGLANG_SET_CPU_AFFINITY": "1", +} + +OTHER_ARGS = [ + "--attention-backend", + "ascend", + "--device", + "npu", + "--tp-size", + 2, + "--chunked-prefill-size", + 16384, + "--max-prefill-tokens", + 150000, + "--dtype", + "bfloat16", + "--max-running-requests", + 32, + "--trust-remote-code", + "--mem-fraction-static", + 0.75, + "--cuda-graph-bs", + 1, + 2, + 4, + 8, + 16, + 32, + "--watchdog-timeout", + 9000, +] + + +class TestQwen3(TestNpuAccuracyTestCaseBase): + model = GLM_4_6V_FLASH_MODEL_PATH + envs = ENVS + other_args = OTHER_ARGS + accuracy = 0.711 + datasets = ["mmmu"] + few_shot_num = 0 + generation_config = {"max_tokens": 65536, "temperature": 1.0} + eval_batch_size = 64 + + def test_mmmu(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py b/test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py new file mode 100644 index 000000000..9ba8bff31 --- /dev/null +++ b/test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py @@ -0,0 +1,72 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_7_FLASH_MODEL_PATH +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="accuracy testcase", +) + +ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_BUFFSIZE": "1000", + "HCCL_OP_EXPANSION_MODE": "AIV", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "SGLANG_SET_CPU_AFFINITY": "1", +} + +OTHER_ARGS = [ + "--attention-backend", + "ascend", + "--device", + "npu", + "--tp-size", + 2, + "--chunked-prefill-size", + 16384, + "--max-prefill-tokens", + 150000, + "--dtype", + "bfloat16", + "--max-running-requests", + 32, + "--trust-remote-code", + "--mem-fraction-static", + 0.75, + "--cuda-graph-bs", + 1, + 2, + 4, + 8, + 16, + 32, + "--watchdog-timeout", + 9000, +] + + +class TestNPUDeepSeek_V3_2_8P_AIME2025(TestNpuAccuracyTestCaseBase): + + model = GLM_4_7_FLASH_MODEL_PATH + envs = ENVS + other_args = OTHER_ARGS + accuracy = 0.916 + datasets = ["aime25"] + few_shot_num = 0 + generation_config = {"max_tokens": 65536, "temperature": 1.0} + eval_batch_size = 64 + + def test_aime2025(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py b/test/registered/ascend/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py new file mode 100644 index 000000000..c76b0348e --- /dev/null +++ b/test/registered/ascend/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py @@ -0,0 +1,86 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + QWEN3_32B_EAGLE_MODEL_PATH, + QWEN3_32B_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_32B_ENVS = { + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200", +} + +QWEN3_32B_OTHER_ARGS = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--max-running-requests", + 64, + "--disable-radix-cache", + "--speculative-draft-model-quantization", + "unquant", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 65536, + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + QWEN3_32B_EAGLE_MODEL_PATH, + "--speculative-num-steps", + 4, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 5, + "--tp-size", + 16, + "--mem-fraction-static", + 0.72, + "--cuda-graph-bs", + 64, + "--dtype", + "bfloat16", +] + + +class TestQwen32B_GPQA(TestNpuAccuracyTestCaseBase): + model = QWEN3_32B_MODEL_PATH + envs = QWEN3_32B_ENVS + other_args = QWEN3_32B_OTHER_ARGS + accuracy = 0.516 + datasets = ["gpqa_diamond"] + few_shot_num = 0 + eval_batch_size = 64 + generation_config = {"max_tokens": 40000, "temperature": 1.0} + + def test_accuracy(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py b/test/registered/ascend/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py new file mode 100644 index 000000000..b24b05fdf --- /dev/null +++ b/test/registered/ascend/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py @@ -0,0 +1,98 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + QWEN3_6_27B_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_6_27B_64K_PREFIX_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "SGLANG_SET_CPU_AFFINITY": "1", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "ASCEND_USE_FIA": "1", + "GDN_ATTN_BACKEND_TRITON": "1", +} + +QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [ + "--tp-size", + 2, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--chunked-prefill-size", + 32768, + "--max-prefill-tokens", + 32768, + "--mamba-scheduler-strategy", + "extra_buffer", + "--trust-remote-code", + "--max-running-requests", + 20, + "--max-mamba-cache-size", + 120, + "--mem-fraction-static", + 0.8, + "--cuda-graph-bs", + 1, + 2, + 4, + 8, + 10, + 12, + 16, + 18, + 20, + "--enable-prefill-delayer", + "--prefill-delayer-queue-min-ratio", + 0.5, + "--prefill-delayer-max-delay-ms", + 30000, + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, +] + + +class TestNPUQwen3_6_27B_2P_In64k_Out1k_Prefix90_gpqa(TestNpuAccuracyTestCaseBase): + model = QWEN3_6_27B_MODEL_PATH + envs = QWEN3_6_27B_64K_PREFIX_ENVS + other_args = QWEN3_6_27B_64K_PREFIX_OTHER_ARGS + accuracy = 0.878 + datasets = ["gpqa_diamond"] + few_shot_num = 0 + eval_batch_size = 64 + generation_config = {"max_tokens": 81920, "temperature": 1.0} + + def test_gpqa(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py b/test/registered/ascend/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py new file mode 100644 index 000000000..8b8032e05 --- /dev/null +++ b/test/registered/ascend/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py @@ -0,0 +1,96 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + QWEN3_6_35B_A3B_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="full-2-npu-a3", + nightly=True, + disabled="performance testcase", +) + +QWEN3_6_35B_A3B_3K5_1K5_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_BUFFSIZE": "100", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", + "ASCEND_USE_FIA": "1", + "GDN_ATTN_BACKEND_TRITON": "1", +} + +QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS = [ + "--tp-size", + 2, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 131072, + "--disable-radix-cache", + "--trust-remote-code", + "--enable-prefill-delayer", + "--max-running-requests", + 4, + "--max-mamba-cache-size", + 4, + "--mem-fraction-static", + 0.7, + "--cuda-graph-bs", + 1, + 2, + 3, + 4, + "--enable-multimodal", + "--mm-attention-backend", + "ascend_attn", + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, +] + + +class TestNPUQwen3_6_35BA3B_1P_In3k5_Out1k5_aime26(TestNpuAccuracyTestCaseBase): + model = QWEN3_6_35B_A3B_MODEL_PATH + envs = QWEN3_6_35B_A3B_3K5_1K5_ENVS + other_args = QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS + accuracy = 0.927 + datasets = ["aime26"] + few_shot_num = 0 + eval_batch_size = 4 + generation_config = { + "max_tokens": 131072, + "temperature": 0.2, + "repetition_penalty": 1.08, + } + + def test_aime26(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py b/test/registered/ascend/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py new file mode 100644 index 000000000..a2cb4980f --- /dev/null +++ b/test/registered/ascend/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py @@ -0,0 +1,104 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + QWEN3_OMNI_30B_A3B_THINKING_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +ENVS = { + "ASCEND_LAUNCH_BLOCKING": "0", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200", + "HCCL_BUFFSIZE": "400", +} + +OTHER_ARGS = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--max-running-requests", + 162, + "--disable-radix-cache", + # "--speculative-draft-model-quantization", + # "unquant", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 35000, + # "--speculative-algorithm", + # "EAGLE3", + # "--speculative-draft-model-path", + # QWEN3_A3B_EAGLE_MODEL_PATH, + # "--speculative-num-steps", + # 3, + # "--speculative-eagle-topk", + # 1, + # "--speculative-num-draft-tokens", + # 4, + "--tp-size", + 2, + "--mem-fraction-static", + 0.87, + "--cuda-graph-bs", + 1, + 5, + 15, + 40, + 70, + 100, + 120, + 130, + 140, + 146, + 150, + 154, + 156, + 158, + 160, + 162, + "--dtype", + "bfloat16", +] + + +class TestQwen3(TestNpuAccuracyTestCaseBase): + model = QWEN3_OMNI_30B_A3B_THINKING_MODEL_PATH + envs = ENVS + other_args = OTHER_ARGS + accuracy = 0.576 + datasets = ["mmmu"] + few_shot_num = 0 + generation_config = {"max_tokens": 65536, "temperature": 1.0} + eval_batch_size = 64 + + def test_mmmu(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py b/test/registered/ascend/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py new file mode 100644 index 000000000..04fb99519 --- /dev/null +++ b/test/registered/ascend/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py @@ -0,0 +1,102 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + QWEN3_VL_30B_A3B_THINKING_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +ENVS = { + "ASCEND_LAUNCH_BLOCKING": "0", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200", + "HCCL_BUFFSIZE": "400", +} + +OTHER_ARGS = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--max-running-requests", + 162, + "--disable-radix-cache", + # "--speculative-draft-model-quantization", + # "unquant", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 35000, + # "--speculative-algorithm", + # "EAGLE3", + # "--speculative-draft-model-path", + # QWEN3_A3B_EAGLE_MODEL_PATH, + # "--speculative-num-steps", + # 3, + # "--speculative-eagle-topk", + # 1, + # "--speculative-num-draft-tokens", + # 4, + "--tp-size", + 2, + "--mem-fraction-static", + 0.87, + "--cuda-graph-bs", + 1, + 5, + 15, + 40, + 70, + 100, + 120, + 130, + 140, + 146, + 150, + 154, + 156, + 158, + 160, + 162, + "--dtype", + "bfloat16", +] + + +class TestQwen3(TestNpuAccuracyTestCaseBase): + model = QWEN3_VL_30B_A3B_THINKING_MODEL_PATH + envs = ENVS + other_args = OTHER_ARGS + accuracy = 0.76 + datasets = ["mmmu"] + few_shot_num = 0 + generation_config = {"max_tokens": 65536, "temperature": 1.0} + eval_batch_size = 64 + + def test_mmmu(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py b/test/registered/ascend/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py new file mode 100644 index 000000000..fe49f2631 --- /dev/null +++ b/test/registered/ascend/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py @@ -0,0 +1,86 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + QWEN3_VL_8B_THINKING_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +ENVS = { + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_ENABLE_SPEC_V2": "1", +} + +OTHER_ARGS = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--max-running-requests", + 16, + "--max-prefill-tokens", + 16384, + "--disable-radix-cache", + "--chunked-prefill-size", + -1, + "--tp-size", + 2, + "--mem-fraction-static", + 0.894, + "--cuda-graph-bs", + 1, + 5, + 15, + 16, + "--dtype", + "bfloat16", + # "--speculative-draft-model-quantization", + # "unquant", + # "--speculative-algorithm", + # "EAGLE3", + # "--speculative-draft-model-path", + # QWEN3_8B_EAGLE_MODEL_PATH, + # "--speculative-num-steps", + # 4, + # "--speculative-eagle-topk", + # 1, + # "--speculative-num-draft-tokens", + # 5, +] + + +class TestQwen3(TestNpuAccuracyTestCaseBase): + model = QWEN3_VL_8B_THINKING_MODEL_PATH + envs = ENVS + other_args = OTHER_ARGS + accuracy = 0.741 + datasets = ["mmmu"] + few_shot_num = 0 + generation_config = {"max_tokens": 65536, "temperature": 1.0} + eval_batch_size = 16 + + def test_mmmu(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py b/test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py new file mode 100644 index 000000000..1894dd8a9 --- /dev/null +++ b/test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py @@ -0,0 +1,198 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyMultiNodePdSepTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + GLM_5_1_W4A8_MODEL_PATH, + TestNpuPerfMultiNodePdSepTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +GLM_5_1_PD_SEP_PREFILL_ENVS = { + "SGLANG_SET_CPU_AFFINITY": "1", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "1200", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "1200", + "HCCL_BUFFSIZE": "1200", + "DEEPEP_NORMAL_LONG_SEQ_ROUND": "72", + "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "1024", + "DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ": "1", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "TASK_QUEUE_ENABLE": "2", + "ENABLE_PROFILING": "0", + "HCCL_SOCKET_IFNAME": NIC_NAME, + "GLOO_SOCKET_IFNAME": NIC_NAME, +} + +GLM_5_1_PD_SEP_DECODE_ENVS = { + "SGLANG_SET_CPU_AFFINITY": "1", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "1200", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "1200", + "SGLANG_SPEC_ENABLE_OVERLAP_REFLOW": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "HCCL_BUFFSIZE": "200", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "16", + "TASK_QUEUE_ENABLE": "0", + "HCCL_SOCKET_IFNAME": NIC_NAME, + "GLOO_SOCKET_IFNAME": NIC_NAME, +} + +GLM_5_1_PD_SEP_PREFILL_ARGS = [ + "--disaggregation-mode", + "prefill", + "--tp-size", + 4, + "--nnodes", + 2, + "--mem-fraction-static", + 0.72, + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--disaggregation-transfer-backend", + "ascend", + "--max-running-requests", + 16, + "--served-model-name", + "glm-5", + "--chunked-prefill-size", + 16384, + "--max-prefill-tokens", + 180000, + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "normal", + "--disable-shared-experts-fusion", + "--disable-cuda-graph", + "--dtype", + "bfloat16", + "--speculative-draft-model-quantization", + "unquant", + "--enable-nsa-prefill-context-parallel", + "--nsa-prefill-cp-mode", + "in-seq-split", + "--attn-cp-size", + 4, + "--enable-dp-lm-head", + "--moe-dense-tp", + 1, + "--pp-size", + 8, +] + +GLM_5_1_PD_SEP_DECODE_ARGS = [ + "--disaggregation-mode", + "decode", + "--tp-size", + 32, + "--nnodes", + 2, + "--dp-size", + 32, + "--enable-dp-attention", + "--ep-size", + 32, + "--mem-fraction-static", + 0.85, + "--max-running-requests", + 32, + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--served-model-name", + "glm-5", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "low_latency", + "--cuda-graph-bs", + 1, + 2, + 3, + "--disaggregation-transfer-backend", + "ascend", + "--watchdog-timeout", + 9000, + "--context-length", + 180000, + "--tokenizer-worker-num", + 16, + "--prefill-round-robin-balance", + "--disable-shared-experts-fusion", + "--dtype", + "bfloat16", + "--load-balance-method", + "round_robin", + "--speculative-draft-model-quantization", + "unquant", +] + +GLM_5_1_PD_SEP_MODEL_CONFIG = { + "model_path": GLM_5_1_W4A8_MODEL_PATH, + "prefill_args": GLM_5_1_PD_SEP_PREFILL_ARGS, + "decode_args": GLM_5_1_PD_SEP_DECODE_ARGS, + "prefill_envs": GLM_5_1_PD_SEP_PREFILL_ENVS, + "decode_envs": GLM_5_1_PD_SEP_DECODE_ENVS, + "router_args": ["--policy", "round_robin"], + "router_envs": {}, +} + + +class TestNPUGLM5_1_W4A8_PD_SEP_AIME2026(TestNpuAccuracyMultiNodePdSepTestCaseBase): + """Test NPU accuracy for GLM-5.1-w4a8 PD separation on AIME2026""" + + model_config = GLM_5_1_PD_SEP_MODEL_CONFIG + accuracy = 0.953 + datasets = ["aime26"] + eval_batch_size = 64 + generation_config = {"max_tokens": 65536, "temperature": 1.0} + + def test_npu_glm5_1_w4a8_pd_sep_aime2026(self): + """Run NPU accuracy test for GLM-5.1-w4a8 PD separation on AIME2026""" + self.run_accuracy() + + +class TestNPUGLM5_1_W4A8_PD_SEP_In3k5_Out1k5(TestNpuPerfMultiNodePdSepTestCaseBase): + """Test NPU performance for GLM-5.1-w4a8 PD separation 4 nodes in3k5 out1k5""" + + model_config = GLM_5_1_PD_SEP_MODEL_CONFIG + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + dataset_name = "random" + max_concurrency = 1 + num_prompts = 1 + input_len = 65536 + output_len = 1024 + random_range_ratio = 1 + tpot = 50 + output_token_throughput = 16 + + def test_npu_glm5_1_w4a8_pd_sep_in3k5_out1k5(self): + """Run NPU performance test for GLM-5.1-w4a8 PD separation""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py b/test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py new file mode 100644 index 000000000..f31529a02 --- /dev/null +++ b/test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py @@ -0,0 +1,125 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyMultiNodePdMixTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + KIMI_K2_6_EAGLE3_MODEL_PATH, + KIMI_K2_6_W4A8_MODEL_PATH, + TestNpuPerfMultiNodePdMixTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=1800, + suite="nightly-8-npu-a3", + nightly=True, + disabled="Currently it is executed by the npu performance workflow.", +) + +ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "SGLANG_SET_CPU_AFFINITY": "1", + "STREAMS_PER_DEVICE": "32", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64", + "HCCL_BUFFSIZE": "4400", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "HCCL_SOCKET_IFNAME": NIC_NAME, + "GLOO_SOCKET_IFNAME": NIC_NAME, +} + +OTHER_ARGS = [ + "--trust-remote-code", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--dtype", + "bfloat16", + "--tp-size", + 32, + "--nnodes", + 2, + "--mem-fraction-static", + 0.55, + "--max-running-requests", + 32, + "--chunked-prefill-size", + 262144, + "--context-length", + 75000, + "--enable-multimodal", + "--mm-attention-backend", + "ascend_attn", + "--sampling-backend", + "ascend", + "--enable-dp-attention", + "--dp-size", + 32, + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "auto", + "--cuda-graph-bs", + 1, + "--disable-radix-cache", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + KIMI_K2_6_EAGLE3_MODEL_PATH, + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--speculative-draft-model-quantization", + "unquant", +] + +MODEL_CONFIG = { + "model_path": KIMI_K2_6_W4A8_MODEL_PATH, + "other_args": OTHER_ARGS, + "node_envs": ENVS, +} + + +class TestNPUKimiK2_6_W4A8_16P_AIME2025(TestNpuAccuracyMultiNodePdMixTestCaseBase): + + model_config = MODEL_CONFIG + accuracy = 0.961 + datasets = ["aime25"] + few_shot_num = 0 + eval_batch_size = 64 + generation_config = {"max_tokens": 65536, "temperature": 1.0} + + def test_aime2025(self): + self.run_accuracy() + + +class TestNPUKimiK2_6_W4A8_16P_In64k_Out1k_100ms(TestNpuPerfMultiNodePdMixTestCaseBase): + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + model_config = MODEL_CONFIG + dataset_name = "random" + max_concurrency = 32 + num_prompts = 32 + input_len = 64000 + output_len = 1000 + random_range_ratio = 1 + tpot = 100 + output_token_throughput = 160 + + def test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms(self): + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py b/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py new file mode 100644 index 000000000..0b7efbf89 --- /dev/null +++ b/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py @@ -0,0 +1,132 @@ +import os +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + BENCHMARK_TOOL_DEFAULT, + MINIMAX_M2_5_EAGLE3_MODEL_PATH, + MINIMAX_M2_5_W8A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="npu-performance", + nightly=True, +) + +MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "TASK_QUEUE_ENABLE": "1", + "ASCEND_USE_FIA": "1", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_NPU_FUSED_MOE_MODE": "2", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "140000", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "HCCL_BUFFSIZE": "1024", + "SGLANG_EXTERNAL_MODEL_PACKAGE": "custom_eagle3", + "PYTHONPATH": f"{MINIMAX_M2_5_EAGLE3_MODEL_PATH}:{os.environ.get('PYTHONPATH', '')}", +} + +MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS = [ + "--tp-size", + 8, + "--mem-fraction-static", + 0.63, + "--max-running-requests", + 26, + "--reasoning-parser", + "minimax-append-think", + "--tool-call-parser", + "minimax-m2", + "--enable-prefill-delayer", + "--prefill-max-requests", + 10, + "--chunked-prefill-size", + 67072, + "--max-prefill-token", + 67000, + "--cuda-graph-bs", + 2, + 4, + 8, + 12, + 16, + 18, + 20, + 22, + 24, + 26, + "--moe-a2a-backend", + "ascend_fuseep", + "--deepep-mode", + "auto", + "--quantization", + "modelslim", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + MINIMAX_M2_5_EAGLE3_MODEL_PATH, + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--speculative-draft-model-quantization", + "unquant", + "--dtype", + "bfloat16", + "--trust-remote-code", +] + + +class TestNPUMiniMaxM2_5W8A8_4P_In64k_Out1k_Prefix90_50ms( + TestNpuPerformanceTestCaseBase +): + """MiniMax-M2.5-w8a8 4p (4 cards) 64k input 1k output with 90% prefix cache performance test""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + model = MINIMAX_M2_5_W8A8_MODEL_PATH + other_args = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS + envs = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS + dataset_name = "generated-shared-prefix" + max_concurrency = 26 + num_prompts = 104 + input_len = 65536 + output_len = 1024 + random_range_ratio = 1 + repeat_rate = 0.9 + tpot = 50 + output_token_throughput = 390.5839 + request_rate = float("inf") + + def test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms(self): + """Run MiniMax-M2.5-w8a8 4p 64k/1k prefix90 performance test""" + self.run_throughput() + + +class TestNPUMiniMaxM2_5_W8A8_4P_In3k5_Out1k5_GPQA(TestNpuAccuracyTestCaseBase): + model = MINIMAX_M2_5_W8A8_MODEL_PATH + other_args = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS + envs = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS + accuracy = 0.852 + datasets = ["gpqa_diamond"] + few_shot_num = 0 + generation_config = {"max_tokens": 65536, "temperature": 1.0} + max_concurrency = 64 + + def test_accuracy(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py b/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py new file mode 100644 index 000000000..219531538 --- /dev/null +++ b/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py @@ -0,0 +1,132 @@ +import os +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + MINIMAX_M2_5_EAGLE3_MODEL_PATH, + MINIMAX_M2_5_W8A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="full-16-npu-a3", + nightly=True, + disabled="performance testcase", +) + +MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "TASK_QUEUE_ENABLE": "1", + "HCCL_BUFFSIZE": "1024", + "ASCEND_USE_FIA": "1", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_NPU_FUSED_MOE_MODE": "2", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "204800", + "PYTHONPATH": f"{MINIMAX_M2_5_EAGLE3_MODEL_PATH}:{os.environ.get('PYTHONPATH', '')}", + "SGLANG_EXTERNAL_MODEL_PACKAGE": "custom_eagle3", +} + +MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS = [ + "--tp-size", + 16, + "--enable-dp-attention", + "--dp-size", + 16, + "--mem-fraction-static", + 0.75, + "--max-running-requests", + 320, + "--disable-radix-cache", + "--reasoning-parser", + "minimax-append-think", + "--tool-call-parser", + "minimax-m2", + "--prefill-delayer-max-delay-passes", + 500, + "--enable-prefill-delayer", + "--chunked-prefill-size", + -1, + "--max-prefill-token", + 8192, + "--cuda-graph-bs", + 1, + 2, + 4, + 8, + 12, + 16, + 20, + "--moe-a2a-backend", + "ascend_fuseep", + "--deepep-mode", + "auto", + "--quantization", + "modelslim", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + MINIMAX_M2_5_EAGLE3_MODEL_PATH, + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--speculative-draft-model-quantization", + "unquant", + "--dtype", + "bfloat16", +] + + +class TestNPUMiniMaxM2_5_W8A8_8P_In3k5_Out1k5_HighThroughput( + TestNpuPerformanceTestCaseBase +): + """Test NPU performance for MiniMax-M2.5-w8a8 8p single node high throughput in3k5 out1k5""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = MINIMAX_M2_5_W8A8_MODEL_PATH + other_args = MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS + envs = MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS + dataset_name = "random" + max_concurrency = 320 + num_prompts = 1280 + input_len = 3500 + output_len = 1500 + random_range_ratio = 1 + tpot = 50 + output_token_throughput = 5717.58 + + def test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_high_throughput(self): + """Run NPU performance test for MiniMax-M2.5-w8a8 high throughput""" + self.run_throughput() + + +class TestNPUMiniMaxM2_5_W8A8_8P_In3k5_Out1k5_GPQA(TestNpuAccuracyTestCaseBase): + model = MINIMAX_M2_5_W8A8_MODEL_PATH + envs = MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS + other_args = MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS + accuracy = 0.852 + datasets = ["gpqa_diamond"] + few_shot_num = 0 + generation_config = {"max_tokens": 65536, "temperature": 1.0} + max_concurrency = 64 + + def test_accuracy(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py b/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py new file mode 100644 index 000000000..4a16dc24c --- /dev/null +++ b/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py @@ -0,0 +1,122 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + QWEN3_8B_EAGLE_MODEL_PATH, + QWEN3_8B_W8A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_8B_ENVS = { + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50", +} + +QWEN3_8B_OTHER_ARGS = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--max-running-requests", + 70, + "--max-prefill-tokens", + 16384, + "--disable-radix-cache", + "--chunked-prefill-size", + 16384, + "--tp-size", + 1, + "--mem-fraction-static", + 0.85, + "--cuda-graph-bs", + 8, + 12, + 24, + 36, + 48, + 51, + 55, + 60, + 63, + 64, + 66, + 68, + 70, + "--dtype", + "bfloat16", + "--speculative-draft-model-quantization", + "unquant", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + QWEN3_8B_EAGLE_MODEL_PATH, + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, +] + + +class TestQwen8B(TestNpuPerformanceTestCaseBase): + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_8B_W8A8_MODEL_PATH + other_args = QWEN3_8B_OTHER_ARGS + envs = QWEN3_8B_ENVS + dataset_name = "random" + max_concurrency = 64 + num_prompts = 256 + input_len = 3500 + output_len = 1500 + random_range_ratio = 1 + tpot = 37 + output_token_throughput = 1586 + + def test_qwen3_8b(self): + self.run_throughput() + + +class TestQwen8B_gpqa(TestNpuAccuracyTestCaseBase): + model = QWEN3_8B_W8A8_MODEL_PATH + envs = QWEN3_8B_ENVS + other_args = QWEN3_8B_OTHER_ARGS + accuracy = 0.4444 + datasets = ["gpqa_diamond"] + few_shot_num = 0 + eval_batch_size = 64 + generation_config = {"max_tokens": 40000, "temperature": 1.0} + + def test_accuracy(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py b/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py new file mode 100644 index 000000000..769a1cddb --- /dev/null +++ b/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py @@ -0,0 +1,108 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + QWEN3_8B_EAGLE_MODEL_PATH, + QWEN3_8B_W8A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_8B_ENVS = { + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_ENABLE_SPEC_V2": "1", +} + +QWEN3_8B_OTHER_ARGS = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--max-running-requests", + 16, + "--max-prefill-tokens", + 16384, + "--disable-radix-cache", + "--chunked-prefill-size", + -1, + "--tp-size", + 2, + "--mem-fraction-static", + 0.894, + "--cuda-graph-bs", + 1, + 5, + 15, + 16, + "--dtype", + "bfloat16", + "--speculative-draft-model-quantization", + "unquant", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + QWEN3_8B_EAGLE_MODEL_PATH, + "--speculative-num-steps", + 4, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 5, +] + + +class TestQwen8B(TestNpuPerformanceTestCaseBase): + max_attempts = 5 + model = QWEN3_8B_W8A8_MODEL_PATH + other_args = QWEN3_8B_OTHER_ARGS + envs = QWEN3_8B_ENVS + dataset_name = "random" + max_concurrency = 16 + num_prompts = 16 + input_len = 6144 + output_len = 1500 + random_range_ratio = 1 + tpot = 11.79 + output_token_throughput = 930 + + def test_qwen3_8b(self): + self.run_throughput() + + +class TestQwen8B_gpqa(TestNpuAccuracyTestCaseBase): + model = QWEN3_8B_W8A8_MODEL_PATH + envs = QWEN3_8B_ENVS + other_args = QWEN3_8B_OTHER_ARGS + accuracy = 0.4444 + datasets = ["gpqa_diamond"] + few_shot_num = 0 + generation_config = {"max_tokens": 32768, "temperature": 1.0} + eval_batch_size = 16 + + def test_accuracy(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py b/test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py new file mode 100644 index 000000000..1d5acf55e --- /dev/null +++ b/test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py @@ -0,0 +1,127 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH, + QWEN3_A3B_EAGLE_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_30B_A3B_ENVS = { + "ASCEND_LAUNCH_BLOCKING": "0", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200", + "HCCL_BUFFSIZE": "400", +} + +QWEN3_30B_A3B_OTHER_ARGS = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--max-running-requests", + 162, + "--disable-radix-cache", + "--speculative-draft-model-quantization", + "unquant", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 35000, + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + QWEN3_A3B_EAGLE_MODEL_PATH, + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--tp-size", + 2, + "--mem-fraction-static", + 0.87, + "--cuda-graph-bs", + 1, + 5, + 15, + 40, + 70, + 100, + 120, + 130, + 140, + 146, + 150, + 154, + 156, + 158, + 160, + 162, + "--dtype", + "bfloat16", +] + + +class TestQwen30B(TestNpuPerformanceTestCaseBase): + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH + other_args = QWEN3_30B_A3B_OTHER_ARGS + envs = QWEN3_30B_A3B_ENVS + dataset_name = "random" + max_concurrency = 160 + num_prompts = int(max_concurrency) * 4 + input_len = 3500 + output_len = 1500 + random_range_ratio = 1 + tpot = 50 + output_token_throughput = 3200 + + def test_qwen3_30b(self): + self.run_throughput() + + +class TestQwen30B_A3B_aime25(TestNpuAccuracyTestCaseBase): + model = QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH + envs = QWEN3_30B_A3B_ENVS + other_args = QWEN3_30B_A3B_OTHER_ARGS + accuracy = 0.613 + datasets = ["aime25"] + few_shot_num = 0 + generation_config = {"max_tokens": 32768, "temperature": 1.0} + eval_batch_size = 16 + + def test_accuracy(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py b/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py new file mode 100644 index 000000000..675f7c832 --- /dev/null +++ b/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py @@ -0,0 +1,91 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + QWEN3_32B_EAGLE_MODEL_PATH, + QWEN3_32B_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_32B_ENVS = { + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200", +} + +QWEN3_32B_OTHER_ARGS = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--max-running-requests", + 1, + "--disable-radix-cache", + "--speculative-draft-model-quantization", + "unquant", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 65536, + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + QWEN3_32B_EAGLE_MODEL_PATH, + "--speculative-num-steps", + 4, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 5, + "--tp-size", + 16, + "--mem-fraction-static", + 0.72, + "--cuda-graph-bs", + 1, + "--dtype", + "bfloat16", +] + + +class TestQwen32B(TestNpuPerformanceTestCaseBase): + benchmark_tool = BENCHMARK_TOOL_DEFAULT + aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_32B_MODEL_PATH + other_args = QWEN3_32B_OTHER_ARGS + envs = QWEN3_32B_ENVS + dataset_name = "random" + max_concurrency = 1 + num_prompts = 1 + input_len = 18000 + output_len = 4000 + random_range_ratio = 1 + tpot = 6 + output_token_throughput = 171 + + def test_qwen3_32b(self): + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py b/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py new file mode 100644 index 000000000..7bf2ac958 --- /dev/null +++ b/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py @@ -0,0 +1,135 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + DEFAULT_URL_FOR_TEST, + QWEN3_32B_EAGLE_MODEL_PATH, + QWEN3_32B_W8A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_32B_ENVS = { + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100", + "SGLANG_NPU_USE_DEEPGEMM": "1", +} + +QWEN3_32B_OTHER_ARGS = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--max-running-requests", + 101, + "--disable-radix-cache", + "--speculative-draft-model-quantization", + "unquant", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 35000, + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + QWEN3_32B_EAGLE_MODEL_PATH, + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--tp-size", + 4, + "--mem-fraction-static", + 0.845, + "--cuda-graph-bs", + 16, + 32, + 64, + 72, + 88, + 90, + 92, + 94, + 96, + 97, + 98, + 99, + 100, + 101, + "--dtype", + "bfloat16", +] + + +class TestQwen32B_GPQA(TestNpuAccuracyTestCaseBase): + """Test NPU accuracy for Qwen3-32B-W8A8 on qpqa""" + + model = QWEN3_32B_W8A8_MODEL_PATH + other_args = QWEN3_32B_OTHER_ARGS + envs = QWEN3_32B_ENVS + accuracy = 0.516 + datasets = ["gpqa_diamond"] + few_shot_num = 0 + eval_batch_size = 64 + generation_config = {"max_tokens": 40000, "temperature": 1.0} + + @classmethod + def tearDownClass(cls): + pass + + def test_qwen3_32b_qpqa(self): + """Run NPU accuracy test for Qwen3-32B-W8A8 on qpqa""" + self.run_accuracy() + + +class TestQwen32B(TestNpuPerformanceTestCaseBase): + base_url = DEFAULT_URL_FOR_TEST + aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_32B_W8A8_MODEL_PATH + other_args = QWEN3_32B_OTHER_ARGS + envs = QWEN3_32B_ENVS + dataset_name = "random" + max_concurrency = 100 + num_prompts = 400 + input_len = 3584 + output_len = 1536 + random_range_ratio = 1 + tpot = 50 + output_token_throughput = 1600 + + @classmethod + def setUpClass(cls): + pass + + def test_qwen3_32b(self): + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py b/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py new file mode 100644 index 000000000..6a7cb216c --- /dev/null +++ b/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py @@ -0,0 +1,124 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + QWEN3_32B_EAGLE_MODEL_PATH, + QWEN3_32B_W8A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_32B_ENVS = { + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100", + "SGLANG_NPU_USE_DEEPGEMM": "1", +} + +QWEN3_32B_OTHER_ARGS = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--max-running-requests", + 101, + "--disable-radix-cache", + "--speculative-draft-model-quantization", + "unquant", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 35000, + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + QWEN3_32B_EAGLE_MODEL_PATH, + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--tp-size", + 4, + "--mem-fraction-static", + 0.845, + "--cuda-graph-bs", + 16, + 32, + 64, + 72, + 88, + 90, + 92, + 94, + 96, + 97, + 98, + 99, + 100, + 101, + "--dtype", + "bfloat16", +] + + +class TestQwen32B(TestNpuPerformanceTestCaseBase): + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_32B_W8A8_MODEL_PATH + other_args = QWEN3_32B_OTHER_ARGS + envs = QWEN3_32B_ENVS + dataset_name = "random" + max_concurrency = 100 + num_prompts = 400 + input_len = 3584 + output_len = 1536 + random_range_ratio = 1 + tpot = 50 + output_token_throughput = 1600 + + def test_qwen3_32b(self): + self.run_throughput() + + +class TestQwen32B_mmlupro(TestNpuAccuracyTestCaseBase): + model = QWEN3_32B_W8A8_MODEL_PATH + envs = QWEN3_32B_ENVS + other_args = QWEN3_32B_OTHER_ARGS + accuracy = 0.4949 + datasets = ["gpqa_diamond"] + few_shot_num = 0 + eval_batch_size = 64 + generation_config = {"max_tokens": 40000, "temperature": 1.0} + + def test_accuracy(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py new file mode 100644 index 000000000..b1b2ea62f --- /dev/null +++ b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py @@ -0,0 +1,111 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_MM_CUSTOM_GEN, + BENCHMARK_TOOL_DEFAULT, + QWEN3_6_27B_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="full-2-npu-a3", + nightly=True, + disabled="performance case", +) + +QWEN3_6_27B_1024_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_VIT_ENABLE_CUDA_GRAPH": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_NPU_PROFILING": "1", + "SGLANG_NPU_PROFILING_STAGE": "prefill", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "150", + "ASCEND_USE_FIA": "1", +} + +QWEN3_6_27B_1024_OTHER_ARGS = [ + "--tp-size", + 2, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 52000, + "--disable-radix-cache", + "--trust-remote-code", + "--max-running-requests", + 50, + "--max-mamba-cache-size", + 60, + "--mem-fraction-static", + 0.76, + "--cuda-graph-bs", + 2, + 4, + 8, + 16, + 24, + 32, + 40, + 42, + 45, + 50, + "--enable-multimodal", + "--mm-attention-backend", + "ascend_attn", + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--mm-enable-dp-encoder", +] + + +class TestNPUQwen3_6_27B_1P_In1024x1024_30_Out1024_50ms(TestNpuPerformanceTestCaseBase): + """Test NPU performance for Qwen3.6-27B 1p in1024x1024 30 out1024 50ms""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + aisbench_dataset_type = AISBENCHMARK_DATASET_MM_CUSTOM_GEN + model = QWEN3_6_27B_MODEL_PATH + other_args = QWEN3_6_27B_1024_OTHER_ARGS + envs = QWEN3_6_27B_1024_ENVS + dataset_name = "random" + max_concurrency = 48 + num_prompts = 48 + input_len = 30 + output_len = 1024 + random_range_ratio = 1 + image_resolution = "1024x1024" + image_count = 1 + tpot = 50 + output_token_throughput = 800.8 + + def test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms(self): + """Run NPU performance test for Qwen3.6-27B in1024x1024 30 out1024 50ms""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py new file mode 100644 index 000000000..72799ad68 --- /dev/null +++ b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py @@ -0,0 +1,108 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_MM_CUSTOM_GEN, + BENCHMARK_TOOL_DEFAULT, + QWEN3_6_27B_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="full-2-npu-a3", + nightly=True, + disabled="performance testcase", +) + +QWEN3_6_27B_1080P_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_VIT_ENABLE_CUDA_GRAPH": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_NPU_PROFILING": "1", + "SGLANG_NPU_PROFILING_STAGE": "prefill", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "150", + "ASCEND_USE_FIA": "1", +} + +QWEN3_6_27B_1080P_OTHER_ARGS = [ + "--tp-size", + 2, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 48000, + "--disable-radix-cache", + "--trust-remote-code", + "--max-running-requests", + 30, + "--max-mamba-cache-size", + 40, + "--mem-fraction-static", + 0.76, + "--cuda-graph-bs", + 2, + 4, + 8, + 16, + 24, + 28, + 30, + "--enable-multimodal", + "--mm-attention-backend", + "ascend_attn", + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--mm-enable-dp-encoder", +] + + +class TestNPUQwen3_6_27B_1P_In1080p_30_Out256_50ms(TestNpuPerformanceTestCaseBase): + """Test NPU performance for Qwen3.6-27B 1p in1080p 30 out256 50ms""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + aisbench_dataset_type = AISBENCHMARK_DATASET_MM_CUSTOM_GEN + model = QWEN3_6_27B_MODEL_PATH + other_args = QWEN3_6_27B_1080P_OTHER_ARGS + envs = QWEN3_6_27B_1080P_ENVS + dataset_name = "random" + max_concurrency = 30 + num_prompts = 120 + input_len = 30 + output_len = 256 + random_range_ratio = 1 + image_resolution = "1920x1080" + image_count = 1 + tpot = 50 + output_token_throughput = 226 + + def test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms(self): + """Run NPU performance test for Qwen3.6-27B in1080p 30 out256 50ms""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py new file mode 100644 index 000000000..76c20a6d1 --- /dev/null +++ b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py @@ -0,0 +1,106 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + BENCHMARK_TOOL_DEFAULT, + QWEN3_6_27B_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_6_27B_64K_PREFIX_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "SGLANG_SET_CPU_AFFINITY": "1", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "ASCEND_USE_FIA": "1", + "GDN_ATTN_BACKEND_TRITON": "1", +} + +QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [ + "--tp-size", + 2, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--chunked-prefill-size", + 32768, + "--max-prefill-tokens", + 32768, + "--mamba-scheduler-strategy", + "extra_buffer", + "--trust-remote-code", + "--max-running-requests", + 20, + "--max-mamba-cache-size", + 120, + "--mem-fraction-static", + 0.8, + "--cuda-graph-bs", + 1, + 2, + 4, + 8, + 10, + 12, + 16, + 18, + 20, + "--enable-prefill-delayer", + "--prefill-delayer-queue-min-ratio", + 0.5, + "--prefill-delayer-max-delay-ms", + 30000, + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, +] + + +class TestNPUQwen3_6_27B_2P_In64k_Out1k_Prefix90_50ms(TestNpuPerformanceTestCaseBase): + """Test NPU performance for Qwen3.6-27B-w8a8 2p in64k out1k prefix90 50ms""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + model = QWEN3_6_27B_MODEL_PATH + other_args = QWEN3_6_27B_64K_PREFIX_OTHER_ARGS + envs = QWEN3_6_27B_64K_PREFIX_ENVS + dataset_name = "generated-shared-prefix" + max_concurrency = 20 + num_prompts = 80 + input_len = 64000 + output_len = 1000 + random_range_ratio = 1 + repeat_rate = 0.9 + request_rate = float("inf") + tpot = 50 + output_token_throughput = 225 + + def test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms(self): + """Run NPU performance test for Qwen3.6-27B-w8a8 in64k out1k prefix90 50ms""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py new file mode 100644 index 000000000..3eca45f15 --- /dev/null +++ b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py @@ -0,0 +1,120 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + QWEN3_6_27B_W8A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="full-2-npu-a3", + nightly=True, + disabled="performance testcase", +) + +QWEN3_6_27B_3K5_1K5_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "130", + "ASCEND_USE_FIA": "1", +} + +QWEN3_6_27B_3K5_1K5_OTHER_ARGS = [ + "--tp-size", + 2, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 60000, + "--disable-radix-cache", + "--trust-remote-code", + "--max-running-requests", + 64, + "--max-mamba-cache-size", + 74, + "--mem-fraction-static", + 0.7, + "--cuda-graph-bs", + 2, + 8, + 16, + 32, + 48, + 64, + "--enable-multimodal", + "--quantization", + "modelslim", + "--mm-attention-backend", + "ascend_attn", + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, +] + + +class TestNPUQwen3_6_27B_1P_In3k5_Out1k5_50ms(TestNpuPerformanceTestCaseBase): + """Test NPU performance for Qwen3.6-27B-w8a8 1p in3k5 out1k5 50ms""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_6_27B_W8A8_MODEL_PATH + other_args = QWEN3_6_27B_3K5_1K5_OTHER_ARGS + envs = QWEN3_6_27B_3K5_1K5_ENVS + dataset_name = "random" + max_concurrency = 54 + num_prompts = 216 + input_len = 3500 + output_len = 1500 + random_range_ratio = 1 + tpot = 50 + output_token_throughput = 786.69 + + def test_npu_qwen3_6_27b_1p_in3k5_out1k5_50ms(self): + """Run NPU performance test for Qwen3.6-27B-w8a8 in3k5 out1k5 50ms""" + self.run_throughput() + + +class TestNPUQwen3_6_27B_1P_In3k5_Out1k5_gpqa(TestNpuAccuracyTestCaseBase): + model = QWEN3_6_27B_W8A8_MODEL_PATH + envs = QWEN3_6_27B_3K5_1K5_ENVS + other_args = QWEN3_6_27B_3K5_1K5_OTHER_ARGS + accuracy = 0.855 + datasets = ["gpqa_diamond"] + few_shot_num = 0 + eval_batch_size = 8 + generation_config = {"max_tokens": 81920, "temperature": 1.0} + + def test_accuracy(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py new file mode 100644 index 000000000..80b42a302 --- /dev/null +++ b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py @@ -0,0 +1,94 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + QWEN3_6_27B_W8A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_6_27B_128K_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "20", + "ASCEND_USE_FIA": "1", +} + +QWEN3_6_27B_128K_OTHER_ARGS = [ + "--tp-size", + 4, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 74000, + "--disable-radix-cache", + "--trust-remote-code", + "--max-running-requests", + 6, + "--max-mamba-cache-size", + 7, + "--mem-fraction-static", + 0.63, + "--cuda-graph-bs", + 1, + 2, + 4, + 5, + 6, + "--enable-multimodal", + "--quantization", + "modelslim", + "--mm-attention-backend", + "ascend_attn", + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", +] + + +class TestNPUQwen3_6_27B_2P_In128k_Out1k_50ms(TestNpuPerformanceTestCaseBase): + """Test NPU performance for Qwen3.6-27B-w8a8 2p in128k out1k 50ms""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_6_27B_W8A8_MODEL_PATH + other_args = QWEN3_6_27B_128K_OTHER_ARGS + envs = QWEN3_6_27B_128K_ENVS + dataset_name = "random" + max_concurrency = 4 + num_prompts = 16 + input_len = 128000 + output_len = 1000 + random_range_ratio = 1 + tpot = 50 + output_token_throughput = 41.39 + + def test_npu_qwen3_6_27b_2p_in128k_out1k_50ms(self): + """Run NPU performance test for Qwen3.6-27B-w8a8 in128k out1k 50ms""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py new file mode 100644 index 000000000..b869342af --- /dev/null +++ b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py @@ -0,0 +1,104 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + QWEN3_6_27B_W8A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_6_27B_16K_1k_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30", + "ASCEND_USE_FIA": "1", +} + +QWEN3_6_27B_16K_1k_OTHER_ARGS = [ + "--tp-size", + 4, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 50000, + "--disable-radix-cache", + "--trust-remote-code", + "--max-running-requests", + 28, + "--max-mamba-cache-size", + 50, + "--mem-fraction-static", + 0.7, + "--cuda-graph-bs", + 2, + 8, + 12, + 16, + 20, + 24, + 28, + "--enable-multimodal", + "--quantization", + "modelslim", + "--mm-attention-backend", + "ascend_attn", + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, +] + + +class TestNPUQwen3_6_27B_2P_In16k_Out1k_50ms(TestNpuPerformanceTestCaseBase): + """Test NPU performance for Qwen3.6-27B-w8a8 2p in16k out1k 50ms""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_6_27B_W8A8_MODEL_PATH + other_args = QWEN3_6_27B_16K_1k_OTHER_ARGS + envs = QWEN3_6_27B_16K_1k_ENVS + dataset_name = "random" + max_concurrency = 28 + num_prompts = 112 + input_len = 16000 + output_len = 1000 + random_range_ratio = 1 + tpot = 50 + output_token_throughput = 426.1 + + def test_npu_qwen3_6_27b_2p_in16k_out1k_50ms(self): + """Run NPU performance test for Qwen3.6-27B-w8a8 in16k out1k 50ms""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py new file mode 100644 index 000000000..36b8de097 --- /dev/null +++ b/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py @@ -0,0 +1,100 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + QWEN3_6_27B_W8A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_6_27B_64K_1K_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30", + "ASCEND_USE_FIA": "1", +} + +QWEN3_6_27B_64K_1K_OTHER_ARGS = [ + "--tp-size", + 4, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 50000, + "--disable-radix-cache", + "--trust-remote-code", + "--max-running-requests", + 28, + "--max-mamba-cache-size", + 50, + "--mem-fraction-static", + 0.7, + "--cuda-graph-bs", + 2, + 4, + 6, + "--enable-multimodal", + "--quantization", + "modelslim", + "--mm-attention-backend", + "ascend_attn", + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, +] + + +class TestNPUQwen3_6_27B_2P_In64k_Out1k_50ms(TestNpuPerformanceTestCaseBase): + """Test NPU performance for Qwen3.6-27B-w8a8 2p in64k out1k 50ms""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_6_27B_W8A8_MODEL_PATH + other_args = QWEN3_6_27B_64K_1K_OTHER_ARGS + envs = QWEN3_6_27B_64K_1K_ENVS + dataset_name = "random" + max_concurrency = 6 + num_prompts = 24 + input_len = 64000 + output_len = 1000 + random_range_ratio = 1 + tpot = 50 + output_token_throughput = 122.6 + + def test_npu_qwen3_6_27b_2p_in64k_out1k_50ms(self): + """Run NPU performance test for Qwen3.6-27B-w8a8 in64k out1k 50ms""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py new file mode 100644 index 000000000..3a5676f06 --- /dev/null +++ b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py @@ -0,0 +1,99 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + QWEN3_6_35B_A3B_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_6_35B_A3B_128K_1K_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "ASCEND_USE_FIA": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "20", +} + +QWEN3_6_35B_A3B_128K_1K_OTHER_ARGS = [ + "--tp-size", + 2, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--chunked-prefill-size", + -1, + "--max-total-tokens", + 600000, + "--max-prefill-tokens", + 65536, + "--disable-radix-cache", + "--trust-remote-code", + "--enable-prefill-delayer", + "--max-running-requests", + 4, + "--max-mamba-cache-size", + 12, + "--mem-fraction-static", + 0.6, + "--max-mamba-cache-size", + 20, + "--disable-cuda-graph", + "--enable-multimodal", + "--mm-attention-backend", + "ascend_attn", + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, +] + + +class TestNPUQwen3_6_35BA3B_1P_In128k_Out1k_50ms(TestNpuPerformanceTestCaseBase): + """Test NPU performance for Qwen3.6-35B-A3B 1p in128k out1k 50ms""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_6_35B_A3B_MODEL_PATH + other_args = QWEN3_6_35B_A3B_128K_1K_OTHER_ARGS + envs = QWEN3_6_35B_A3B_128K_1K_ENVS + dataset_name = "random" + max_concurrency = 4 + num_prompts = 16 + input_len = 128000 + output_len = 1000 + random_range_ratio = 1 + tpot = 50 + output_token_throughput = 60.57 + + def test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms(self): + """Run NPU performance test for Qwen3.6-35B-A3B in128k out1k 50ms""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py new file mode 100644 index 000000000..5f3aacca9 --- /dev/null +++ b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py @@ -0,0 +1,108 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + BENCHMARK_TOOL_DEFAULT, + QWEN3_6_35B_A3B_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_6_35B_A3B_128K_PREFIX_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "ASCEND_USE_FIA": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30", +} + +QWEN3_6_35B_A3B_128K_PREFIX_OTHER_ARGS = [ + "--tp-size", + 2, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--chunked-prefill-size", + 16384, + "--max-prefill-tokens", + 65536, + "--trust-remote-code", + "--enable-prefill-delayer", + "--mamba-scheduler-strategy", + "extra_buffer", + "--max-running-requests", + 103, + "--max-mamba-cache-size", + 85, + "--mem-fraction-static", + 0.85, + "--cuda-graph-bs", + 2, + 4, + 8, + 16, + 32, + 48, + 64, + 80, + 96, + 103, + "--enable-multimodal", + "--mm-attention-backend", + "ascend_attn", + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, +] + + +class TestNPUQwen3_6_35BA3B_1P_In128k_Out1k_Prefix90_50ms( + TestNpuPerformanceTestCaseBase +): + """Test NPU performance for Qwen3.6-35B-A3B 1p in128k out1k prefix90 50ms""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + model = QWEN3_6_35B_A3B_MODEL_PATH + other_args = QWEN3_6_35B_A3B_128K_PREFIX_OTHER_ARGS + envs = QWEN3_6_35B_A3B_128K_PREFIX_ENVS + dataset_name = "generated-shared-prefix" + max_concurrency = 103 + num_prompts = 412 + input_len = 64000 + output_len = 1000 + random_range_ratio = 1 + repeat_rate = 0.9 + tpot = 50 + request_rate = float("inf") + output_token_throughput = 308.2 + + def test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms(self): + """Run NPU performance test for Qwen3.6-35B-A3B in128k out1k prefix90 50ms""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py new file mode 100644 index 000000000..6813843ce --- /dev/null +++ b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py @@ -0,0 +1,103 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + QWEN3_6_35B_A3B_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="full-2-npu-a3", + nightly=True, + disabled="performance testcase", +) + +QWEN3_6_35B_A3B_3K5_1K5_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_BUFFSIZE": "800", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", + "ASCEND_USE_FIA": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50", +} + +QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS = [ + "--tp-size", + 2, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 35000, + "--disable-radix-cache", + "--trust-remote-code", + "--enable-prefill-delayer", + "--max-running-requests", + 110, + "--max-mamba-cache-size", + 115, + "--mem-fraction-static", + 0.78, + "--cuda-graph-bs", + 4, + 16, + 32, + 64, + 84, + 105, + 110, + "--enable-multimodal", + "--mm-attention-backend", + "ascend_attn", + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, +] + + +class TestNPUQwen3_6_35BA3B_1P_In3k5_Out1k5_50ms(TestNpuPerformanceTestCaseBase): + """Test NPU performance for Qwen3.6-35B-A3B 1p in3k5 out1k5 50ms""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_6_35B_A3B_MODEL_PATH + other_args = QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS + envs = QWEN3_6_35B_A3B_3K5_1K5_ENVS + dataset_name = "random" + max_concurrency = 110 + num_prompts = 440 + input_len = 3500 + output_len = 1500 + random_range_ratio = 1 + tpot = 50 + output_token_throughput = 2031.71 + + def test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms(self): + """Run NPU performance test for Qwen3.6-35B-A3B in3k5 out1k5 50ms""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py new file mode 100644 index 000000000..fa6b79f3c --- /dev/null +++ b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py @@ -0,0 +1,103 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + QWEN3_6_35B_A3B_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="full-2-npu-a3", + nightly=True, + disabled="performance testcase", +) + +QWEN3_6_35B_A3B_64K_1K_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "ASCEND_USE_FIA": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "1", +} + +QWEN3_6_35B_A3B_64K_1K_OTHER_ARGS = [ + "--tp-size", + 2, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--chunked-prefill-size", + -1, + "--max-total-tokens", + 600000, + "--max-prefill-tokens", + 65536, + "--disable-radix-cache", + "--trust-remote-code", + "--enable-prefill-delayer", + "--max-running-requests", + 10, + "--max-mamba-cache-size", + 20, + "--mem-fraction-static", + 0.65, + "--cuda-graph-bs", + 2, + 4, + 8, + 12, + 14, + 16, + "--enable-multimodal", + "--mm-attention-backend", + "ascend_attn", + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, +] + + +class TestNPUQwen3_6_35BA3B_1P_In64k_Out1k_50ms(TestNpuPerformanceTestCaseBase): + """Test NPU performance for Qwen3.6-35B-A3B 1p in64k out1k 50ms""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_6_35B_A3B_MODEL_PATH + other_args = QWEN3_6_35B_A3B_64K_1K_OTHER_ARGS + envs = QWEN3_6_35B_A3B_64K_1K_ENVS + dataset_name = "random" + max_concurrency = 10 + num_prompts = 40 + input_len = 64000 + output_len = 1000 + random_range_ratio = 1 + tpot = 50 + output_token_throughput = 141.72 + + def test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms(self): + """Run NPU performance test for Qwen3.6-35B-A3B in64k out1k 50ms""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py new file mode 100644 index 000000000..86bdd5ed1 --- /dev/null +++ b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py @@ -0,0 +1,141 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + BENCHMARK_TOOL_DEFAULT, + DEFAULT_URL_FOR_TEST, + QWEN3_6_35B_A3B_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_6_35B_A3B_64K_PREFIX_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_BUFFSIZE": "300", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", + "ASCEND_USE_FIA": "1", + "GDN_ATTN_BACKEND_TRITON": "1", +} + +QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [ + "--tp-size", + 2, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 65536, + "--trust-remote-code", + "--enable-prefill-delayer", + "--mamba-scheduler-strategy", + "extra_buffer", + "--max-running-requests", + 42, + "--max-mamba-cache-size", + 210, + "--mem-fraction-static", + 0.71, + "--cuda-graph-bs", + 2, + 8, + 16, + 24, + 32, + 36, + 40, + 42, + "--enable-multimodal", + "--mm-attention-backend", + "ascend_attn", + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, +] + + +class TestNPUQwen3_6_35BA3B_1P_AIME2026(TestNpuAccuracyTestCaseBase): + """Test NPU accuracy for Qwen3.6-35B-A3B 1p on AIME2026""" + + model = QWEN3_6_35B_A3B_MODEL_PATH + other_args = QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS + envs = QWEN3_6_35B_A3B_64K_PREFIX_ENVS + accuracy = 0.927 + datasets = ["aime26"] + few_shot_num = 0 + eval_batch_size = 64 + generation_config = { + "max_tokens": 65536, + "temperature": 0.2, + "repetition_penalty": 1.08, + } + + @classmethod + def tearDownClass(cls): + pass + + def test_npu_qwen3_6_35b_a3b_1p_aime2026(self): + """Run NPU accuracy test for Qwen3.6-35B-A3B on AIME2026""" + self.run_accuracy() + + +class TestNPUQwen3_6_35BA3B_1P_In64k_Out1k_Prefix90_50ms( + TestNpuPerformanceTestCaseBase +): + """Test NPU performance for Qwen3.6-35B-A3B 1p in64k out1k prefix90 50ms""" + + base_url = DEFAULT_URL_FOR_TEST + benchmark_tool = BENCHMARK_TOOL_DEFAULT + model = QWEN3_6_35B_A3B_MODEL_PATH + other_args = QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS + envs = QWEN3_6_35B_A3B_64K_PREFIX_ENVS + dataset_name = "generated-shared-prefix" + max_concurrency = 42 + num_prompts = 42 + input_len = 65536 + output_len = 1024 + random_range_ratio = 1 + repeat_rate = 0.9 + tpot = 50 + request_rate = float("inf") + output_token_throughput = 660 + + @classmethod + def setUpClass(cls): + pass + + def test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms(self): + """Run NPU performance test for Qwen3.6-35B-A3B in64k out1k prefix90 50ms""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py b/test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py new file mode 100644 index 000000000..380507d99 --- /dev/null +++ b/test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py @@ -0,0 +1,135 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + QWEN3_NEXT_80B_A3B_MODEL_PATH, + QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_NEXT_80B_A3B_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "400", + "DEEPEP_NORMAL_LONG_SEQ_ROUND": "10", + "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "2048", + "HCCL_OP_EXPANSION_MODE": "AIV", + "TASK_QUEUE_ENABLE": "1", + "ASCEND_USE_FIA": "1", + "SGLANG_NPU_USE_MULTI_STREAM": "0", + "SGLANG_WARMUP_TIMEOUT": "3600", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "FORCE_DRAFT_MODEL_NON_QUANT": "1", + "HCCL_BUFFSIZE": "2000", + "ZBCCL_LOCAL_MEM_SIZE": "60416", + "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", + "ZBCCL_BOOTSTRAP_URL": "tcp://127.0.0.1:24669", + "ZBCCL_NPU_ALLOC_CONF": "use_vmm_for_static_memory:True", + "ZBCCL_ENABLE_GRAPH": "1", +} + +QWEN3_NEXT_80B_A3B_OTHER_ARGS = [ + "--trust-remote-code", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--page-size", + 128, + "--tp-size", + 4, + "--watchdog-timeout", + 9000, + "--mem-fraction-static", + 0.85, + "--disable-radix-cache", + "--max-prefill-tokens", + 28672, + "--context-length", + 81920, + "--max-total-tokens", + 122304, + "--dp-size", + 2, + "--enable-dp-attention", + "--enable-dp-lm-head", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--speculative-draft-model-quantization", + "unquant", + "--chunked-prefill-size", + -1, + "--max-running-requests", + 16, + "--cuda-graph-bs", + 2, + 4, + 8, + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-draft-model-path", + QWEN3_NEXT_80B_A3B_MODEL_PATH, +] + + +class TestQwen3Next80BA3B(TestNpuPerformanceTestCaseBase): + max_attempts = 5 + model = QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH + other_args = QWEN3_NEXT_80B_A3B_OTHER_ARGS + envs = QWEN3_NEXT_80B_A3B_ENVS + dataset_name = "random" + max_concurrency = 16 + num_prompts = 16 + input_len = 6144 + output_len = 1500 + random_range_ratio = 1 + tpot = 15.62 + + def test_qwen3_next_80b_a3b(self): + self.run_throughput() + + +class TestQwen3Next80BA3B_aime25(TestNpuAccuracyTestCaseBase): + model = QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH + envs = QWEN3_NEXT_80B_A3B_ENVS + other_args = QWEN3_NEXT_80B_A3B_OTHER_ARGS + accuracy = 0.695 + datasets = ["aime25"] + few_shot_num = 0 + generation_config = { + "max_tokens": 65536, + "temperature": 0.7, + "top_p": 0.8, + "top_k": 20, + "extra_body": {"chat_template_kwargs": {"enable_thinking": True}}, + } + max_concurrency = 16 + + def test_aime25(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main()