From ba214ef3d3632109afcf92163da4fa3810054a1b Mon Sep 17 00:00:00 2001 From: Alison Shao <54658187+alisonshao@users.noreply.github.com> Date: Thu, 14 May 2026 16:11:15 -0700 Subject: [PATCH] =?UTF-8?q?ci:=20tag-gated=20nightly=20migration=20?= =?UTF-8?q?=E2=80=94=20foundation=20+=2040=20whole-file=20moves=20(#24725)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: hnyls2002 Co-authored-by: Liangsheng Yin --- .github/workflows/_pr-test-check-changes.yml | 5 + .github/workflows/pr-test-extra.yml | 225 ++++ .github/workflows/pr-test.yml | 17 - .../sglang/test/kits/streaming_session_kit.py | 441 +++++++ .../hybrid_attn_backend_fixture.py | 102 ++ .../test/server_fixtures/ngram_fixture.py | 74 ++ .../test/server_fixtures/pcg_spec_fixture.py | 80 ++ .../server_fixtures/standalone_fixture.py | 114 ++ .../streaming_session_fixture.py | 434 +++++++ scripts/ci/utils/compute_partitions.py | 4 +- .../function_call/test_tool_choice.py | 0 .../quant/test_deepseek_v3_fp4_4gpu_trtllm.py | 93 ++ test/manual/spec/test_spec_ngram_fa3.py | 18 + .../4-gpu-models/test_qwen35_hicache.py | 2 +- .../test_deepseek_v32_indexcache.py | 2 +- .../test_nvidia_nemotron_3_super_bf16.py | 2 +- .../8-gpu-models/test_return_indexer_topk.py | 2 +- .../test_step3p5_flash_chain_mtp.py | 2 +- .../attention/test_hybrid_attn_backend.py | 165 +-- .../test_torch_native_attention_backend.py | 2 +- .../attention/test_triton_sliding_window.py | 2 +- .../registered/backends/test_torch_compile.py | 2 +- test/registered/core/test_gpt_oss_sm120.py | 2 +- .../cp/test_deepseek_v32_cp_single_node.py | 6 +- .../test_disaggregation_hybrid_attention.py | 2 +- .../test_load_weights_from_remote_instance.py | 2 +- test/registered/ep/test_deepep_large.py | 2 +- .../hicache/test_hicache_spec_file_storage.py | 2 +- test/registered/lora/test_lora_drainer.py | 2 +- test/registered/lora/test_lora_eviction.py | 2 +- .../test_lora_gpt_oss_20b_logprob_diff.py | 6 +- .../lora/test_lora_moe_tp_logprob_diff.py | 6 +- ...nemotron_3_super_120b_a12b_logprob_diff.py | 6 +- ...wen3_30b_a3b_instruct_2507_logprob_diff.py | 6 +- .../lora/test_lora_qwen3_5_4b_logprob_diff.py | 6 +- .../lora/test_lora_qwen3_8b_logprob_diff.py | 6 +- .../models/test_compressed_tensors_models.py | 2 +- .../models/test_generation_models.py | 2 +- .../models/test_ministral4_models.py | 6 +- test/registered/models/test_vlm_models.py | 2 +- test/registered/moe/test_cutedsl_moe.py | 2 +- test/registered/moe/test_moe_ep.py | 43 +- test/registered/moe/test_moe_ep_extra.py | 66 + test/registered/observability/test_tracing.py | 2 +- .../perf/test_bench_one_batch_2gpu.py | 2 +- .../perf/test_bench_serving_1gpu_large.py | 2 +- .../perf/test_bench_serving_1gpu_part1.py | 2 +- .../perf/test_bench_serving_1gpu_part2.py | 2 +- .../perf/test_bench_serving_2gpu.py | 2 +- .../test_pcg_with_speculative_decoding.py | 258 +--- ...est_pcg_with_speculative_decoding_extra.py | 81 ++ .../quant/test_deepseek_v3_fp4_4gpu.py | 130 +- .../quant/test_deepseek_v3_fp4_4gpu_extra.py | 87 ++ .../quant/test_fp8_blockwise_gemm.py | 2 +- test/registered/quant/test_fp8_gemm_sm120.py | 2 +- test/registered/quant/test_fp8kv_triton.py | 2 +- .../quant/test_gptqmodel_dynamic.py | 2 +- .../quant/test_w8a8_quantization.py | 2 +- .../rl/test_lora_load_from_tensor.py | 2 +- ...ulti_instance_release_memory_occupation.py | 2 +- .../rl/test_return_routed_experts.py | 2 +- ...test_update_weights_from_disk_blackwell.py | 2 +- .../test_update_weights_from_distributed.py | 2 +- .../rl/test_update_weights_from_tensor.py | 2 +- .../scheduler/test_priority_scheduling.py | 2 +- .../sessions/test_session_control.py | 2 +- .../sessions/test_session_latency.py | 6 +- .../sessions/test_streaming_session.py | 1130 +---------------- .../sessions/test_streaming_session_extra.py | 105 ++ .../sessions/test_streaming_session_swa.py | 170 +-- .../spec/eagle/test_eagle_dp_attention.py | 7 +- .../spec/eagle/test_eagle_infer_a.py | 2 +- test/registered/spec/test_spec_ngram.py | 19 + ...e_decoding.py => test_spec_ngram_extra.py} | 91 +- test/registered/spec/test_spec_standalone.py | 27 + .../spec/test_spec_standalone_extra.py | 34 + .../test_standalone_speculative_decoding.py | 223 ---- test/run_suite.py | 10 +- 78 files changed, 2254 insertions(+), 2131 deletions(-) create mode 100644 .github/workflows/pr-test-extra.yml create mode 100644 python/sglang/test/kits/streaming_session_kit.py create mode 100644 python/sglang/test/server_fixtures/hybrid_attn_backend_fixture.py create mode 100644 python/sglang/test/server_fixtures/ngram_fixture.py create mode 100644 python/sglang/test/server_fixtures/pcg_spec_fixture.py create mode 100644 python/sglang/test/server_fixtures/standalone_fixture.py create mode 100644 python/sglang/test/server_fixtures/streaming_session_fixture.py rename test/{registered => manual}/openai_server/function_call/test_tool_choice.py (100%) create mode 100644 test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py create mode 100644 test/manual/spec/test_spec_ngram_fa3.py create mode 100644 test/registered/moe/test_moe_ep_extra.py create mode 100644 test/registered/piecewise_cuda_graph/test_pcg_with_speculative_decoding_extra.py create mode 100644 test/registered/quant/test_deepseek_v3_fp4_4gpu_extra.py create mode 100644 test/registered/sessions/test_streaming_session_extra.py create mode 100644 test/registered/spec/test_spec_ngram.py rename test/registered/spec/{test_ngram_speculative_decoding.py => test_spec_ngram_extra.py} (53%) create mode 100644 test/registered/spec/test_spec_standalone.py create mode 100644 test/registered/spec/test_spec_standalone_extra.py delete mode 100644 test/registered/spec/test_standalone_speculative_decoding.py diff --git a/.github/workflows/_pr-test-check-changes.yml b/.github/workflows/_pr-test-check-changes.yml index df04e35ff..eef3d88c8 100644 --- a/.github/workflows/_pr-test-check-changes.yml +++ b/.github/workflows/_pr-test-check-changes.yml @@ -21,6 +21,10 @@ on: force_continue_on_error: type: boolean default: false + pr_test_yml: + description: 'Workflow YAML whose stage `run_timeout_minutes` drives partition sizing.' + type: string + default: '.github/workflows/pr-test.yml' outputs: main_package: value: ${{ jobs.run.outputs.main_package }} @@ -274,6 +278,7 @@ jobs: python3 scripts/ci/utils/compute_partitions.py \ --full-parallel ${{ steps.parallel-mode.outputs.full }} \ --partition-model-file /tmp/partition-model.json \ + --pr-test-yml ${{ inputs.pr_test_yml }} \ >> "$GITHUB_OUTPUT" - name: Set B200 runner tag diff --git a/.github/workflows/pr-test-extra.yml b/.github/workflows/pr-test-extra.yml new file mode 100644 index 000000000..92602cfef --- /dev/null +++ b/.github/workflows/pr-test-extra.yml @@ -0,0 +1,225 @@ +name: PR Test Extra +# Label-gated CI for nightly-class tests opted into a per-PR run. +# +# Adds runtime to a PR only when the author asks for it: pull_request +# events bail unless the PR carries the `run-ci-extra` label. The same job +# graph runs unconditionally on workflow_dispatch / workflow_call so it +# can be triggered manually or chained from another workflow. +# +# Stages: extra-a (1-/2-gpu) and extra-b (4-/8-gpu) caller stubs reuse +# `_pr-test-stage.yml` and `_pr-test-check-changes.yml` from pr-test.yml. + +run-name: ${{ inputs.target_stage && (inputs.pr_head_sha && format('[{0}] {1}', inputs.target_stage, inputs.pr_head_sha) || format('[{0}]', inputs.target_stage)) || '' }} + +on: + pull_request: + branches: [main] + workflow_dispatch: + inputs: + target_stage: + description: "Specific stage to run (optional, for quick testing)" + required: false + type: string + default: "" + force_continue_on_error: + description: "Force continue-on-error (test scheduled CI behavior)" + required: false + type: boolean + default: false + pr_head_sha: + description: "PR head SHA to checkout (for /rerun-stage on fork PRs)" + required: false + type: string + default: "" + include_wheel_build: + description: "When set with target_stage, also run sgl-kernel-build-wheels so the target stage uses the freshly-built kernel (for /rerun-stage on PRs that modify sgl-kernel/)" + required: false + type: boolean + default: false + test_parallel_dispatch: + description: "Test parallel dispatch behavior (simulates scheduled run)" + required: false + type: boolean + default: false + workflow_call: + inputs: + git_ref: + description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' + required: false + type: string + default: '' + run_all_tests: + description: "Run all tests (for releasing or testing purpose)" + required: false + type: boolean + default: false + skip_stage_health_check: + description: "Skip stage health check fast-fail (e.g. for release branch cuts)" + required: false + type: boolean + default: false + +concurrency: + group: pr-test-extra-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}-${{ inputs.pr_head_sha || 'current' }}-${{ inputs.target_stage || inputs.git_ref || 'all' }} + cancel-in-progress: ${{ github.event_name != 'workflow_call' }} + +env: + SGLANG_IS_IN_CI: true + SGLANG_CUDA_COREDUMP: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true + SKIP_STAGE_HEALTH_CHECK: ${{ inputs.skip_stage_health_check == true && 'true' || 'false' }} + FORCE_REBUILD_DEEPEP: '1' + PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }} + USE_VENV: false + +permissions: + actions: write + contents: read + issues: read + pull-requests: read + +jobs: + # =============================================== check changes ==================================================== + # Label gate: pull_request events only proceed when the PR carries BOTH + # `run-ci` and `run-ci-extra` labels — `run-ci` is the basic-CI prerequisite + # (matching pr-test.yml's pr-gate `require-run-ci`) and `run-ci-extra` is the + # explicit opt-in to this workflow. Other event types + # (workflow_dispatch / workflow_call) always run. When this job is + # skipped by the gate, every downstream caller stub naturally skips + # because its needs do not resolve. + check-changes: + if: | + github.event_name != 'pull_request' || + ( + contains(github.event.pull_request.labels.*.name, 'run-ci') && + contains(github.event.pull_request.labels.*.name, 'run-ci-extra') + ) + uses: ./.github/workflows/_pr-test-check-changes.yml + with: + pr_head_sha: ${{ inputs.pr_head_sha || '' }} + git_ref: ${{ inputs.git_ref || '' }} + target_stage: ${{ inputs.target_stage || '' }} + include_wheel_build: ${{ inputs.include_wheel_build == true }} + run_all_tests: ${{ inputs.run_all_tests == true }} + force_continue_on_error: ${{ inputs.force_continue_on_error == true }} + pr_test_yml: '.github/workflows/pr-test-extra.yml' + secrets: inherit + + # =============================================== sgl-kernel ==================================================== + sgl-kernel-build-wheels: + needs: check-changes + if: | + always() && + needs.check-changes.result == 'success' && + needs.check-changes.outputs.sgl_kernel == 'true' && + (!inputs.target_stage || inputs.include_wheel_build) + uses: ./.github/workflows/_pr-test-sgl-kernel-build.yml + with: + runs_on: x64-kernel-build-node + job_display_name: Build Wheel + pr_head_sha: ${{ inputs.pr_head_sha || '' }} + git_ref: ${{ inputs.git_ref || '' }} + skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} + secrets: inherit + + # =============================================== extra-a (1-/2-gpu) =============================================== + extra-a-test-1-gpu-small: + needs: [check-changes, sgl-kernel-build-wheels] + if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: extra-a-test-1-gpu-small + runner_config: 1-gpu-small + runs_on: 1-gpu-5090 + check_changes: ${{ toJson(needs.check-changes.outputs) }} + caller_inputs: ${{ toJson(inputs) }} + partitions: ${{ needs.check-changes.outputs.partitions }} + run_timeout_minutes: '60' + secrets: inherit + + extra-a-test-1-gpu-large: + needs: [check-changes, sgl-kernel-build-wheels] + if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: extra-a-test-1-gpu-large + runner_config: 1-gpu-large + runs_on: 1-gpu-h100 + check_changes: ${{ toJson(needs.check-changes.outputs) }} + caller_inputs: ${{ toJson(inputs) }} + partitions: ${{ needs.check-changes.outputs.partitions }} + run_timeout_minutes: '60' + timeout_per_file: '1800' + secrets: inherit + + extra-a-test-2-gpu-large: + needs: [check-changes, sgl-kernel-build-wheels] + if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: extra-a-test-2-gpu-large + runner_config: 2-gpu-large + runs_on: 2-gpu-h100 + check_changes: ${{ toJson(needs.check-changes.outputs) }} + caller_inputs: ${{ toJson(inputs) }} + partitions: ${{ needs.check-changes.outputs.partitions }} + run_timeout_minutes: '60' + secrets: inherit + + # =============================================== extra-b (4-/8-gpu) =============================================== + extra-b-test-4-gpu-h100: + needs: [check-changes, sgl-kernel-build-wheels] + if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: extra-b-test-4-gpu-h100 + runner_config: 4-gpu-h100 + runs_on: 4-gpu-h100 + check_changes: ${{ toJson(needs.check-changes.outputs) }} + caller_inputs: ${{ toJson(inputs) }} + partitions: ${{ needs.check-changes.outputs.partitions }} + run_timeout_minutes: '60' + secrets: inherit + + extra-b-test-4-gpu-b200: + needs: [check-changes, sgl-kernel-build-wheels] + if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: extra-b-test-4-gpu-b200 + runner_config: 4-gpu-b200 + runs_on: ${{ needs.check-changes.outputs.b200_runner }} + check_changes: ${{ toJson(needs.check-changes.outputs) }} + caller_inputs: ${{ toJson(inputs) }} + partitions: ${{ needs.check-changes.outputs.partitions }} + run_timeout_minutes: '60' + timeout_per_file: '1800' + secrets: inherit + + extra-b-test-8-gpu-h200: + needs: [check-changes, sgl-kernel-build-wheels] + if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: extra-b-test-8-gpu-h200 + runner_config: 8-gpu-h200 + runs_on: 8-gpu-h200 + check_changes: ${{ toJson(needs.check-changes.outputs) }} + caller_inputs: ${{ toJson(inputs) }} + partitions: ${{ needs.check-changes.outputs.partitions }} + run_timeout_minutes: '60' + secrets: inherit + + extra-b-test-deepep-8-gpu-h200: + needs: [check-changes, sgl-kernel-build-wheels] + if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: extra-b-test-deepep-8-gpu-h200 + runner_config: deepep-8-gpu-h200 + runs_on: 8-gpu-h200-deepep + check_changes: ${{ toJson(needs.check-changes.outputs) }} + caller_inputs: ${{ toJson(inputs) }} + partitions: ${{ needs.check-changes.outputs.partitions }} + run_timeout_minutes: '60' + secrets: inherit diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 84c074139..928e9658a 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -518,22 +518,6 @@ jobs: warmup_server_models: 'lmsys/sglang-ci-dsv3-test:4' secrets: inherit - stage-c-test-deepep-8-gpu-h200: - needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: stage-c-test-deepep-8-gpu-h200 - runner_config: deepep-8-gpu-h200 - runs_on: 8-gpu-h200-deepep - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '45' - warmup_deep_gemm_models: 'deepseek-ai/DeepSeek-V3-0324:8 deepseek-ai/DeepSeek-V3.2:8' - warmup_server_models: 'deepseek-ai/DeepSeek-V3-0324:8' - secrets: inherit - stage-c-test-4-gpu-b200: needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels] if: ${{ !failure() && !cancelled() }} @@ -606,7 +590,6 @@ jobs: stage-c-test-8-gpu-h20, stage-c-test-8-gpu-h200, stage-c-test-deepep-4-gpu-h100, - stage-c-test-deepep-8-gpu-h200, stage-c-test-4-gpu-b200, stage-c-test-dsv4-4-gpu-b200, stage-c-test-dsv4-8-gpu-h200, diff --git a/python/sglang/test/kits/streaming_session_kit.py b/python/sglang/test/kits/streaming_session_kit.py new file mode 100644 index 000000000..0d4e7d6fa --- /dev/null +++ b/python/sglang/test/kits/streaming_session_kit.py @@ -0,0 +1,441 @@ +"""Streaming-session test method mixins. + +Pair these with `StreamingSessionServerBase` (from sglang.test.server_fixtures.streaming_session_fixture) +to assemble a concrete test class. Per the sglang fixture/kit split: +the fixture only launches the server; the kit owns the `test_*` methods. + +- `StreamingSessionKitMixin`: KV-inheritance + chunked-prefill + abort-recovery + + concurrent-logprob/stress test methods. +- `AbortLeakReproKitMixin`: single test method for abort-heavy chunked-prefill leak repro. +""" + +import asyncio +import time + +import requests + +from sglang.test.server_fixtures.streaming_session_fixture import ( + _abort_repro_run_all, + _concurrent_logprob_run, + _stress_run_all, +) + + +class StreamingSessionKitMixin: + """Streaming-session KV-inheritance + retract/abort-recovery suite.""" + + # -1 for non-overlap subclasses: the last sampled token isn't committed + # before max_new stops, so slot.kv_committed_len = input + output - 1. + kv_inherit_offset = 0 + + def test_kv_cache_inheritance(self, gen_len=12): + """Each turn's cached_tokens must equal previous turn's prompt+completion + (modulo kv_inherit_offset).""" + chunks = [ + "Let me tell you something about France.", + "The capital of France is", + "The population of the city is", + ] + chunks_ids = [self.tokenizer.encode(x) for x in chunks] + for i in range(1, len(chunks_ids)): + if chunks_ids[i][0] == self.tokenizer.bos_token_id: + chunks_ids[i] = chunks_ids[i][1:] + + # === Part 1: streaming session — check KV inheritance === + requests.post(self.base_url + "/flush_cache") + session_id = requests.post( + self.base_url + "/open_session", + json={"capacity_of_str_len": 1000, "streaming": True}, + ).json() + rid = None + + prev_kv_len = 0 + for turn_idx, chunk_ids in enumerate(chunks_ids): + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": chunk_ids, + "session_params": {"id": session_id, "rid": rid}, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + rid = response["meta_info"]["id"] + cached = response["meta_info"]["cached_tokens"] + prompt_tokens = response["meta_info"]["prompt_tokens"] + completion_tokens = response["meta_info"]["completion_tokens"] + + if turn_idx == 0: + # Turn 1: cache flushed, no hit. + self.assertEqual(cached, 0, "Turn 1: clean start, no cache hit") + else: + # Turns 2+: cached_tokens reflects KV inherited from previous turn + # (via inherit_kv_states, not radix tree matching). + expected = prev_kv_len + self.kv_inherit_offset + self.assertEqual( + cached, + expected, + f"Turn {turn_idx + 1}: inherited {cached} != expected {expected}", + ) + prev_kv_len = prompt_tokens + completion_tokens + + # Close the session. + ret = requests.post( + self.base_url + "/close_session", + json={"session_id": session_id}, + ) + self.assertEqual(ret.status_code, 200) + + def test_leak_logprob_concurrent(self) -> None: + """Concurrent multi-session × 3 logprob modes (output / input / none), + watch for KV leak.""" + requests.post(self.base_url + "/flush_cache") + # Output logprob + asyncio.run( + _concurrent_logprob_run(self.base_url, self.tokenizer, return_logprob=True) + ) + # Input logprob (logprob_start_len=0) + asyncio.run( + _concurrent_logprob_run( + self.base_url, + self.tokenizer, + return_logprob=True, + logprob_start_len=0, + ) + ) + # No logprob + asyncio.run(_concurrent_logprob_run(self.base_url, self.tokenizer)) + time.sleep(3) + assert ( + requests.get(self.base_url + "/health").status_code == 200 + ), "Server unhealthy after concurrent logprob sessions." + + def test_stress_concurrent_sessions(self) -> None: + """High concurrency streaming + non-streaming with retract pressure; + scheduler must roll back streaming KV without leaking.""" + requests.post(self.base_url + "/flush_cache") + asyncio.run(_stress_run_all(self.base_url, self.tokenizer)) + + for i in range(3): + ids = self.tokenizer.encode(f"Post-stress cleanup {i}.") + requests.post( + self.base_url + "/generate", + json={ + "input_ids": ids, + "sampling_params": {"temperature": 0, "max_new_tokens": 4}, + }, + ) + + time.sleep(5) + health = requests.get(self.base_url + "/health") + self.assertEqual( + health.status_code, + 200, + "Server unhealthy after concurrent stress test — " + "likely a token leak from retract/mixed-chunk + streaming session.", + ) + + def test_nth_mid_abort_recovery(self) -> None: + """Abort an Nth-turn request mid-decode; session rolls back to last + successful turn.""" + requests.post(self.base_url + "/flush_cache") + + resp = requests.post( + self.base_url + "/open_session", + json={"capacity_of_str_len": 50000, "streaming": True}, + ) + self.assertEqual(resp.status_code, 200) + session_id = resp.json() + + try: + # Turn 1: normal generate to create slot. + ids_1 = self.tokenizer.encode("Tell me a very long story about a wizard.") + resp_1 = requests.post( + self.base_url + "/generate", + json={ + "input_ids": ids_1, + "sampling_params": {"temperature": 0, "max_new_tokens": 16}, + "session_params": {"id": session_id, "rid": None}, + }, + timeout=30, + ) + self.assertEqual(resp_1.status_code, 200, resp_1.text) + data_1 = resp_1.json() + turn_1_total = ( + data_1["meta_info"]["prompt_tokens"] + + data_1["meta_info"]["completion_tokens"] + ) + + # Turn 2: long generate, then abort mid-decode. + ids_2 = self.tokenizer.encode(" Continue the story in great detail.") + + import threading + + result = [None] + + def do_generate(): + r = requests.post( + self.base_url + "/generate", + json={ + "input_ids": ids_2, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 100000, + }, + "session_params": {"id": session_id, "rid": None}, + }, + timeout=60, + ) + result[0] = r + + t = threading.Thread(target=do_generate) + t.start() + time.sleep(0.5) + abort_resp = requests.post( + self.base_url + "/abort_request", + json={"rid": "", "abort_all": True}, + timeout=10, + ) + self.assertEqual(abort_resp.status_code, 200, abort_resp.text) + t.join(timeout=30) + + self.assertIsNotNone(result[0], "Turn 2 should have returned") + data_2 = result[0].json() + self.assertEqual( + data_2["meta_info"]["finish_reason"]["type"], + "abort", + "Turn 2 should be aborted, not finished normally", + ) + + # Turn 3: recovery. Rolls back to turn 1. + ids_3 = self.tokenizer.encode(" What happens next?") + for attempt in range(20): + resp_3 = requests.post( + self.base_url + "/generate", + json={ + "input_ids": ids_3, + "sampling_params": {"temperature": 0, "max_new_tokens": 8}, + "session_params": {"id": session_id, "rid": None}, + }, + timeout=30, + ) + if resp_3.status_code == 200: + break + time.sleep(0.5) + self.assertEqual(resp_3.status_code, 200, resp_3.text) + data_3 = resp_3.json() + # prompt_tokens = turn_1_total + append (BOS stripped). + bos = 1 if ids_3[0] == self.tokenizer.bos_token_id else 0 + expected_prompt_3 = turn_1_total + len(ids_3) - bos + self.assertEqual( + data_3["meta_info"]["prompt_tokens"], + expected_prompt_3, + "prompt_tokens must equal turn_1_total + append (no stale abort context)", + ) + finally: + requests.post( + self.base_url + "/close_session", + json={"session_id": session_id}, + ) + + health = requests.get(self.base_url + "/health", timeout=10) + self.assertEqual(health.status_code, 200) + + def test_first_mid_abort_recovery(self) -> None: + """Abort the very first request mid-decode (no slot yet; ephemeral + slot is created and nuked). Session must still be usable.""" + requests.post(self.base_url + "/flush_cache") + + resp = requests.post( + self.base_url + "/open_session", + json={"capacity_of_str_len": 50000, "streaming": True}, + ) + self.assertEqual(resp.status_code, 200) + session_id = resp.json() + + try: + ids_1 = self.tokenizer.encode("Tell me a very long story about a wizard.") + + import threading + + result = [None] + + def do_generate(): + r = requests.post( + self.base_url + "/generate", + json={ + "input_ids": ids_1, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 100000, + }, + "session_params": {"id": session_id, "rid": None}, + }, + timeout=60, + ) + result[0] = r + + t = threading.Thread(target=do_generate) + t.start() + time.sleep(0.5) + abort_resp = requests.post( + self.base_url + "/abort_request", + json={"rid": "", "abort_all": True}, + timeout=10, + ) + self.assertEqual(abort_resp.status_code, 200, abort_resp.text) + t.join(timeout=30) + + self.assertIsNotNone(result[0], "Turn 1 should have returned") + data_1 = result[0].json() + self.assertEqual( + data_1["meta_info"]["finish_reason"]["type"], + "abort", + "Turn 1 should be aborted, not finished normally", + ) + + # Turn 2: recovery. No inherited context (req_nodes empty). + ids_2 = self.tokenizer.encode("Tell me a short joke.") + for attempt in range(20): + resp_2 = requests.post( + self.base_url + "/generate", + json={ + "input_ids": ids_2, + "sampling_params": {"temperature": 0, "max_new_tokens": 8}, + "session_params": {"id": session_id, "rid": None}, + }, + timeout=30, + ) + if resp_2.status_code == 200: + break + time.sleep(0.5) + self.assertEqual(resp_2.status_code, 200, resp_2.text) + data_2 = resp_2.json() + self.assertEqual( + data_2["meta_info"]["prompt_tokens"], + len(ids_2), + "prompt_tokens must equal turn 2 input only (no inherited context)", + ) + finally: + requests.post( + self.base_url + "/close_session", + json={"session_id": session_id}, + ) + + health = requests.get(self.base_url + "/health", timeout=10) + self.assertEqual(health.status_code, 200) + + def test_preabort_recovery(self) -> None: + """Pre-abort (rejected by create_req) preserves the slot; next turn + inherits correctly.""" + requests.post(self.base_url + "/flush_cache") + + resp = requests.post( + self.base_url + "/open_session", + json={"capacity_of_str_len": 50000, "streaming": True}, + ) + self.assertEqual(resp.status_code, 200) + session_id = resp.json() + + try: + # Turn 1: normal generate to create slot. + ids_1 = self.tokenizer.encode("Tell me a very long story about a wizard.") + resp_1 = requests.post( + self.base_url + "/generate", + json={ + "input_ids": ids_1, + "sampling_params": {"temperature": 0, "max_new_tokens": 16}, + "session_params": {"id": session_id, "rid": None}, + }, + timeout=30, + ) + self.assertEqual(resp_1.status_code, 200, resp_1.text) + data_1 = resp_1.json() + turn_1_total = ( + data_1["meta_info"]["prompt_tokens"] + + data_1["meta_info"]["completion_tokens"] + ) + + # Turn 2: pre-aborted via unsupported offset parameter. + ids_2 = self.tokenizer.encode(" This should be rejected.") + resp_2 = requests.post( + self.base_url + "/generate", + json={ + "input_ids": ids_2, + "sampling_params": {"temperature": 0, "max_new_tokens": 8}, + "session_params": { + "id": session_id, + "rid": None, + "offset": 1, + }, + }, + timeout=30, + ) + self.assertIn(resp_2.status_code, (200, 400), resp_2.text) + + # Turn 3: normal append. Slot should be intact from turn 1. + ids_3 = self.tokenizer.encode(" What happens next?") + resp_3 = requests.post( + self.base_url + "/generate", + json={ + "input_ids": ids_3, + "sampling_params": {"temperature": 0, "max_new_tokens": 8}, + "session_params": {"id": session_id, "rid": None}, + }, + timeout=30, + ) + self.assertEqual(resp_3.status_code, 200, resp_3.text) + data_3 = resp_3.json() + bos = 1 if ids_3[0] == self.tokenizer.bos_token_id else 0 + expected_prompt_3 = turn_1_total + len(ids_3) - bos + self.assertEqual( + data_3["meta_info"]["prompt_tokens"], + expected_prompt_3, + "prompt_tokens must equal turn_1_total + append (slot preserved)", + ) + finally: + requests.post( + self.base_url + "/close_session", + json={"session_id": session_id}, + ) + + health = requests.get(self.base_url + "/health", timeout=10) + self.assertEqual(health.status_code, 200) + + +class AbortLeakReproKitMixin: + """Abort-heavy chunked-prefill leak repro.""" + + def test_abort_heavy_chunked_prefill_does_not_leak(self) -> None: + requests.post(self.base_url + "/flush_cache") + + asyncio.run(_abort_repro_run_all(self.base_url, self.tokenizer)) + + for i in range(3): + ids = self.tokenizer.encode(f"Post-session cleanup request {i}.") + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": ids, + "sampling_params": {"temperature": 0, "max_new_tokens": 4}, + }, + timeout=30, + ) + self.assertEqual(response.status_code, 200, response.text) + + time.sleep(5) + self.assertIsNone( + self.process.poll(), + "Server crashed during abort-heavy streaming session repro.", + ) + + health = requests.get(self.base_url + "/health", timeout=10) + self.assertEqual( + health.status_code, + 200, + "Server unhealthy after abort-heavy streaming session cleanup.", + ) diff --git a/python/sglang/test/server_fixtures/hybrid_attn_backend_fixture.py b/python/sglang/test/server_fixtures/hybrid_attn_backend_fixture.py new file mode 100644 index 000000000..0a184ea80 --- /dev/null +++ b/python/sglang/test/server_fixtures/hybrid_attn_backend_fixture.py @@ -0,0 +1,102 @@ +"""Hybrid attention-backend (FA3 prefill + FlashInfer decode) test fixture. + +Variants combine `TestHybridAttnBackendBase` with their own +`get_server_args()` / `accuracy_threshold` / `speculative_decode` knobs. + +Requires SM 90+ (H100); the base class wraps that in a `skipIf`. +""" + +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.environ import envs +from sglang.srt.utils import get_device_sm, kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TARGET_MODEL_EAGLE, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +GSM_DATASET_PATH = None + +# Default server arguments shared across all hybrid-attn-backend tests +DEFAULT_HYBRID_ATTN_SERVER_ARGS = [ + "--trust-remote-code", + "--cuda-graph-max-bs", + "8", + "--prefill-attention-backend", + "fa3", + "--decode-attention-backend", + "flashinfer", +] + + +@unittest.skipIf(get_device_sm() < 90, "Test requires CUDA SM 90 or higher") +class TestHybridAttnBackendBase(CustomTestCase): + + model = DEFAULT_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + accuracy_threshold = 0.65 # derived tests need to override this + speculative_decode = False + spec_decode_threshold = 2.2 # derived spec decoding tests need to override this + # Appended after DEFAULT_HYBRID_ATTN_SERVER_ARGS in get_server_args. + extra_args: list = [] + + @classmethod + def get_server_args(cls): + return DEFAULT_HYBRID_ATTN_SERVER_ARGS + list(cls.extra_args) + + @classmethod + def setUpClass(cls): + # disable deep gemm precompile to make launch server faster + # please don't do this if you want to make your inference workload faster + with ( + envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.override(False), + envs.SGLANG_ENABLE_JIT_DEEPGEMM.override(False), + ): + if cls.speculative_decode: + model = DEFAULT_TARGET_MODEL_EAGLE + else: + model = cls.model + cls.process = popen_launch_server( + model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=cls.get_server_args(), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + model = DEFAULT_TARGET_MODEL_EAGLE if self.speculative_decode else self.model + args = SimpleNamespace( + base_url=self.base_url, + model=model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=100, + num_threads=128, + ) + metrics = run_eval(args) + print(f"{metrics=}") + + self.assertGreater(metrics["score"], self.accuracy_threshold) + + if self.speculative_decode: + server_info = requests.get(self.base_url + "/server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold) diff --git a/python/sglang/test/server_fixtures/ngram_fixture.py b/python/sglang/test/server_fixtures/ngram_fixture.py new file mode 100644 index 000000000..4caa15d69 --- /dev/null +++ b/python/sglang/test/server_fixtures/ngram_fixture.py @@ -0,0 +1,74 @@ +"""NGRAM speculative-decoding server fixture. + +Variants combine this base with `GSM8KMixin` and override `attention_backend` +(required) plus optional `extra_args` to select a backend / pass extra flags. + +Example: + from sglang.test.server_fixtures.ngram_fixture import NgramServerBase + from sglang.test.kits.eval_accuracy_kit import GSM8KMixin + + class TestNgramSpeculativeDecodingTriton(NgramServerBase, GSM8KMixin): + attention_backend = "triton" + +The base itself is NOT a runnable test (no `test_*` methods until a subclass +mixes in GSM8KMixin), so unittest discovery picks it up as empty. +""" + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TARGET_MODEL_NGRAM, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +DEFAULT_NGRAM_SERVER_ARGS = [ + "--trust-remote-code", + "--cuda-graph-max-bs", + "8", + "--speculative-algorithm", + "NGRAM", + "--speculative-num-draft-tokens", + "16", + "--mem-fraction-static", + 0.8, +] + + +class NgramServerBase(CustomTestCase): + model = DEFAULT_TARGET_MODEL_NGRAM + base_url = DEFAULT_URL_FOR_TEST + gsm8k_accuracy_thres = 0.79 + gsm8k_accept_length_thres = 1.8 + + # Subclasses must set `attention_backend`; `extra_args` is optional. + attention_backend: str = "" + extra_args: list = [] + + @classmethod + def get_server_args(cls): + assert cls.attention_backend, f"{cls.__name__} must set `attention_backend`" + return ( + DEFAULT_NGRAM_SERVER_ARGS + + ["--attention-backend", cls.attention_backend] + + list(cls.extra_args) + ) + + @classmethod + def setUpClass(cls): + # disable deep gemm precompile to make launch server faster + # please don't do this if you want to make your inference workload faster + envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False) + envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=cls.get_server_args(), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) diff --git a/python/sglang/test/server_fixtures/pcg_spec_fixture.py b/python/sglang/test/server_fixtures/pcg_spec_fixture.py new file mode 100644 index 000000000..c0307dfe3 --- /dev/null +++ b/python/sglang/test/server_fixtures/pcg_spec_fixture.py @@ -0,0 +1,80 @@ +"""Piecewise CUDA Graph + speculative decoding test fixture. + +Each variant tests PCG coexisting with one speculative-decoding algorithm +(EAGLE3 / NEXTN / STANDALONE / NGRAM). Variants differ widely on model / +server args / thresholds, so the base only abstracts the common shape: + - launch a server with `server_args` (variant-supplied list) + - run gsm8k, assert `score > accuracy_threshold` + - read `avg_spec_accept_length` from /server_info, assert + `> speedup_threshold` + +Pure mixin (does NOT inherit `TestCase`), so unittest does not collect +the base itself. +""" + +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + popen_launch_server, +) + + +class PCGSpecBase: + # Subclasses must set: + model: str = "" + server_args: list = [] + + # Optional knobs (variant defaults override): + timeout_mult: int = 2 + server_env: dict = None # passed to popen_launch_server `env=...` + accuracy_threshold: float = 0.70 + speedup_threshold: float = 1.5 + max_tokens: int = 512 + thinking_mode: str = "" # set to e.g. "qwen3" if needed + + @classmethod + def setUpClass(cls): + assert ( + cls.model and cls.server_args + ), f"{cls.__name__} must set `model` and `server_args`" + cls.base_url = DEFAULT_URL_FOR_TEST + kwargs = dict( + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * cls.timeout_mult, + other_args=cls.server_args, + ) + if cls.server_env: + kwargs["env"] = cls.server_env + cls.process = popen_launch_server(cls.model, cls.base_url, **kwargs) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + eval_kwargs = dict( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + max_tokens=self.max_tokens, + num_examples=200, + num_threads=200, + ) + if self.thinking_mode: + eval_kwargs["thinking_mode"] = self.thinking_mode + args = SimpleNamespace(**eval_kwargs) + metrics = run_eval(args) + print(metrics) + self.assertGreater(metrics["score"], self.accuracy_threshold) + + server_info = requests.get(self.base_url + "/server_info").json() + avg_spec_accept_length = server_info["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, self.speedup_threshold) diff --git a/python/sglang/test/server_fixtures/standalone_fixture.py b/python/sglang/test/server_fixtures/standalone_fixture.py new file mode 100644 index 000000000..fc0e6f2f0 --- /dev/null +++ b/python/sglang/test/server_fixtures/standalone_fixture.py @@ -0,0 +1,114 @@ +"""STANDALONE speculative-decoding server fixture. + +Variants combine this base with `CustomTestCase` and override class +attributes (`attention_backend`, plus optional `speculative_eagle_topk` / +`speculative_num_draft_tokens` / `enable_spec_v2`) to select a backend +and the V1 / V2 spec engine. + +Pure mixin (does NOT inherit `TestCase`), so unittest does not collect +the base itself. +""" + +from types import SimpleNamespace + +import requests + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_DRAFT_MODEL_STANDALONE, + DEFAULT_TARGET_MODEL_STANDALONE, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + popen_launch_server, +) + +GSM_DATASET_PATH = None + + +class StandaloneServerBase: + model = DEFAULT_TARGET_MODEL_STANDALONE + draft_model = DEFAULT_DRAFT_MODEL_STANDALONE + base_url = DEFAULT_URL_FOR_TEST + accuracy_threshold = 0.69 + spec_decode_threshold = 3.6 + + # Subclasses set these: + attention_backend: str = "" + # V2 defaults; V1 subclasses override to (2, 7, False). + speculative_num_steps: int = 4 + speculative_eagle_topk: int = 1 + speculative_num_draft_tokens: int = 5 + enable_spec_v2: bool = True + + @classmethod + def get_server_args(cls): + assert cls.attention_backend, f"{cls.__name__} must set `attention_backend`" + return [ + "--trust-remote-code", + "--cuda-graph-max-bs", + "8", + "--speculative-algorithm", + "STANDALONE", + "--speculative-draft-model-path", + DEFAULT_DRAFT_MODEL_STANDALONE, + "--speculative-num-steps", + str(cls.speculative_num_steps), + "--speculative-eagle-topk", + str(cls.speculative_eagle_topk), + "--speculative-num-draft-tokens", + str(cls.speculative_num_draft_tokens), + "--mem-fraction-static", + 0.7, + "--attention-backend", + cls.attention_backend, + ] + + @classmethod + def setUpClass(cls): + # disable deep gemm precompile to make launch server faster + # please don't do this if you want to make your inference workload faster + envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False) + envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False) + if not cls.enable_spec_v2: + envs.SGLANG_ENABLE_SPEC_V2.set(False) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=cls.get_server_args(), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + if not cls.enable_spec_v2: + envs.SGLANG_ENABLE_SPEC_V2.clear() + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=100, + num_threads=128, + num_shots=4, + gsm8k_data_path=GSM_DATASET_PATH, + ) + metrics = run_eval(args) + print(f"{metrics=}") + + metric_key = "score" + self.assertGreaterEqual(metrics[metric_key], self.accuracy_threshold) + + server_info = requests.get(self.base_url + "/server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold) diff --git a/python/sglang/test/server_fixtures/streaming_session_fixture.py b/python/sglang/test/server_fixtures/streaming_session_fixture.py new file mode 100644 index 000000000..8ffd111b0 --- /dev/null +++ b/python/sglang/test/server_fixtures/streaming_session_fixture.py @@ -0,0 +1,434 @@ +"""Streaming-session test fixture. + +`TestStreamingSession` is the base class for all streaming-session tests +(default config — Llama-3.1-8B, no spec). Variants in +test_streaming_session.py and test_streaming_session_extra.py inherit +it and only override `setUpClass`. + +Also exports: +- ABORT_REPRO_* constants used by the basic file's abort-leak repro. +- _abort_repro_run_all coroutine reused by the basic file. + +Lives under sglang.test.server_fixtures so siblings under test/registered +can `import` it without sys.path hacks. +""" + +import asyncio +import json +from typing import Any, Optional + +import aiohttp +import requests + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.srt.utils.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +LOGPROB_PROMPTS = [ + "The quick brown fox jumps over the lazy dog.", + "Pack my box with five dozen liquor jugs.", + "How vexingly quick daft zebras jump.", + "Sphinx of black quartz judge my vow.", + "The five boxing wizards jump quickly.", +] + +# Long enough to trigger chunked prefill at 200+ tokens per slice. +LEAK_FILLER = ( + "The quick brown fox jumps over the lazy dog. " + "Pack my box with five dozen liquor jugs. " + "How vexingly quick daft zebras jump. " + "Sphinx of black quartz, judge my vow. " + "The five boxing wizards jump quickly. " + "Jackdaws love my big sphinx of quartz. " + "A wizard's job is to vex chumps quickly in fog. " + "We promptly judged antique ivory buckles for the next prize. " +) * 20 + +ABORT_REPRO_CONTEXT_LEN = 512 +ABORT_REPRO_PAGE_SIZE = 256 +ABORT_REPRO_GEN_LEN = 4 +ABORT_REPRO_SESSIONS = 4 +ABORT_REPRO_WARMUP_TURNS = 1 +ABORT_REPRO_ROUNDS = 8 +ABORT_REPRO_STREAM_TOKENS = 16 +ABORT_REPRO_ABORT_TOKENS = 600 +ABORT_REPRO_NON_STREAMING_TOKENS = 16 +ABORT_REPRO_CHUNKED_PREFILL_SIZE = 4096 + +CONCURRENT_LOGPROB_SESSIONS = 6 +CONCURRENT_LOGPROB_TURNS = 5 +CONCURRENT_LOGPROB_ROUNDS = 10 + +STRESS_NUM_SESSIONS = 8 +STRESS_NUM_NON_STREAMING = 4 +STRESS_NUM_TURNS = 6 +STRESS_GEN_LEN = 16 + + +def _make_token_sized_ids( + tokenizer: Any, prefix: str, min_tokens: int, max_tokens: Optional[int] = None +) -> list[int]: + text = prefix + chunk = " pack quartz wizard sphinx zebra fox " * 16 + token_ids = tokenizer.encode(text) + while len(token_ids) < min_tokens: + text += chunk + token_ids = tokenizer.encode(text) + if max_tokens is not None: + token_ids = token_ids[:max_tokens] + return token_ids + + +async def _abort_repro_generate( + base_url: str, + session: aiohttp.ClientSession, + input_ids: list[int], + max_new_tokens: int, + session_params: Optional[dict[str, Any]] = None, + expect_abort: bool = False, +) -> Optional[dict[str, Any]]: + payload: dict[str, Any] = { + "input_ids": input_ids, + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + } + if session_params: + payload["session_params"] = session_params + + async with session.post(base_url + "/generate", json=payload) as resp: + text = await resp.text() + if expect_abort: + if resp.status == 200: + data = json.loads(text) + finish_reason = data.get("meta_info", {}).get("finish_reason", {}) + assert finish_reason.get("type") == "abort", text + assert "maximum allowed length" in finish_reason.get( + "message", "" + ) or "context length" in finish_reason.get("message", ""), text + return data + assert resp.status == 400, text + assert "maximum allowed length" in text or "context length" in text, text + return None + + assert resp.status == 200, text + data = json.loads(text) + finish_reason = data.get("meta_info", {}).get("finish_reason", {}) + assert finish_reason.get("type") != "abort", text + return data + + +async def _abort_repro_run_all(base_url: str, tokenizer: Any) -> None: + timeout = aiohttp.ClientTimeout(total=300) + async with aiohttp.ClientSession(timeout=timeout) as http: + session_ids = [] + for _ in range(ABORT_REPRO_SESSIONS): + async with http.post( + base_url + "/open_session", + json={"capacity_of_str_len": 50000, "streaming": True}, + ) as resp: + assert resp.status == 200, await resp.text() + session_ids.append(await resp.json()) + + try: + for warmup_turn in range(ABORT_REPRO_WARMUP_TURNS): + warmup_tasks = [] + for session_idx, session_id in enumerate(session_ids): + input_ids = _make_token_sized_ids( + tokenizer, + prefix=f"[warmup={warmup_turn} session={session_idx}]", + min_tokens=ABORT_REPRO_STREAM_TOKENS, + max_tokens=ABORT_REPRO_STREAM_TOKENS + 8, + ) + warmup_tasks.append( + _abort_repro_generate( + base_url, + http, + input_ids, + ABORT_REPRO_GEN_LEN, + session_params={"id": session_id, "rid": None}, + ) + ) + await asyncio.gather(*warmup_tasks) + + for round_idx in range(ABORT_REPRO_ROUNDS): + mixed_tasks = [] + for session_idx, session_id in enumerate(session_ids): + input_ids = _make_token_sized_ids( + tokenizer, + prefix=f"[round={round_idx} ok session={session_idx}]", + min_tokens=ABORT_REPRO_STREAM_TOKENS, + max_tokens=ABORT_REPRO_STREAM_TOKENS + 8, + ) + mixed_tasks.append( + _abort_repro_generate( + base_url, + http, + input_ids, + ABORT_REPRO_GEN_LEN, + session_params={"id": session_id, "rid": None}, + ) + ) + + for ns_idx in range(2): + input_ids = _make_token_sized_ids( + tokenizer, + prefix=f"[round={round_idx} ns={ns_idx}]", + min_tokens=ABORT_REPRO_NON_STREAMING_TOKENS, + max_tokens=ABORT_REPRO_NON_STREAMING_TOKENS + 8, + ) + mixed_tasks.append( + _abort_repro_generate( + base_url, + http, + input_ids, + ABORT_REPRO_GEN_LEN, + ) + ) + await asyncio.gather(*mixed_tasks) + + abort_tasks = [] + for session_idx, session_id in enumerate(session_ids): + input_ids = _make_token_sized_ids( + tokenizer, + prefix=f"[round={round_idx} abort session={session_idx}]", + min_tokens=ABORT_REPRO_ABORT_TOKENS, + ) + abort_tasks.append( + _abort_repro_generate( + base_url, + http, + input_ids, + ABORT_REPRO_GEN_LEN, + session_params={"id": session_id, "rid": None}, + expect_abort=True, + ) + ) + await asyncio.gather(*abort_tasks) + + recovery_tasks = [] + for session_idx, session_id in enumerate(session_ids): + input_ids = _make_token_sized_ids( + tokenizer, + prefix=f"[round={round_idx} recover session={session_idx}]", + min_tokens=ABORT_REPRO_NON_STREAMING_TOKENS, + max_tokens=ABORT_REPRO_NON_STREAMING_TOKENS + 8, + ) + recovery_tasks.append( + _abort_repro_generate( + base_url, + http, + input_ids, + ABORT_REPRO_GEN_LEN, + session_params={"id": session_id, "rid": None}, + ) + ) + recovery_results = await asyncio.gather(*recovery_tasks) + for result in recovery_results: + assert result is not None + assert result["meta_info"]["cached_tokens"] > 0, result + + health = requests.get(base_url + "/health", timeout=10) + if health.status_code != 200: + raise RuntimeError( + f"server unhealthy after round={round_idx}: " + f"{health.status_code} {health.text}" + ) + finally: + for session_id in session_ids: + async with http.post( + base_url + "/close_session", json={"session_id": session_id} + ) as resp: + assert resp.status == 200, await resp.text() + + +async def _async_generate( + base_url: str, + session: aiohttp.ClientSession, + input_ids: list[int], + max_new_tokens: int = 8, + session_params: Optional[dict[str, Any]] = None, + return_logprob: bool = False, + logprob_start_len: Optional[int] = None, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "input_ids": input_ids, + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + } + if session_params: + payload["session_params"] = session_params + if return_logprob: + payload["return_logprob"] = True + if logprob_start_len is not None: + payload["logprob_start_len"] = logprob_start_len + timeout = aiohttp.ClientTimeout(total=300) + async with session.post( + base_url + "/generate", json=payload, timeout=timeout + ) as resp: + assert resp.status == 200, f"Generate failed: {await resp.text()}" + return await resp.json() + + +async def _concurrent_logprob_run(base_url: str, tokenizer: Any, **gen_kwargs) -> None: + """N sessions per round, all requests fired simultaneously per turn so + the running batch has real concurrency (retract can actually kick one). + """ + timeout = aiohttp.ClientTimeout(total=300) + async with aiohttp.ClientSession(timeout=timeout) as http: + for _ in range(CONCURRENT_LOGPROB_ROUNDS): + sids: list[str] = [] + for _ in range(CONCURRENT_LOGPROB_SESSIONS): + async with http.post( + base_url + "/open_session", + json={"capacity_of_str_len": 50000, "streaming": True}, + ) as resp: + assert resp.status == 200 + sids.append(await resp.json()) + + rids: list[Optional[str]] = [None] * CONCURRENT_LOGPROB_SESSIONS + for turn in range(CONCURRENT_LOGPROB_TURNS): + tasks = [] + for s in range(CONCURRENT_LOGPROB_SESSIONS): + text = ( + f"S{s} T{turn}: " + f"{LOGPROB_PROMPTS[turn % len(LOGPROB_PROMPTS)]}" + ) + ids = tokenizer.encode(text) + tasks.append( + _async_generate( + base_url, + http, + ids, + session_params={"id": sids[s], "rid": rids[s]}, + **gen_kwargs, + ) + ) + results = await asyncio.gather(*tasks) + for s in range(CONCURRENT_LOGPROB_SESSIONS): + rids[s] = results[s]["meta_info"]["id"] + + for sid in sids: + async with http.post( + base_url + "/close_session", json={"session_id": sid} + ) as resp: + assert resp.status == 200 + + +async def _stress_run_all(base_url: str, tokenizer: Any) -> None: + """Streaming + non-streaming mixed batches under retract pressure. + Long prompts (~200+ tokens) trigger chunked prefill so retract can + interrupt mid-extend. + """ + timeout = aiohttp.ClientTimeout(total=300) + async with aiohttp.ClientSession(timeout=timeout) as http: + sids: list[str] = [] + for _ in range(STRESS_NUM_SESSIONS): + async with http.post( + base_url + "/open_session", + json={"capacity_of_str_len": 50000, "streaming": True}, + ) as resp: + assert resp.status == 200 + sids.append(await resp.json()) + + rids: list[Optional[str]] = [None] * STRESS_NUM_SESSIONS + for turn in range(STRESS_NUM_TURNS): + tasks = [] + # Streaming requests — long prompts to trigger chunked prefill. + for s in range(STRESS_NUM_SESSIONS): + offset = (s * STRESS_NUM_TURNS + turn) * 200 + text = ( + f"Session {s} turn {turn}: " f"{LEAK_FILLER[offset : offset + 800]}" + ) + ids = tokenizer.encode(text) + tasks.append( + _async_generate( + base_url, + http, + ids, + max_new_tokens=STRESS_GEN_LEN, + session_params={"id": sids[s], "rid": rids[s]}, + ) + ) + + # Non-streaming requests interleaved. + for ns in range(STRESS_NUM_NON_STREAMING): + text = ( + f"Non-streaming {ns} turn {turn}: " + f"{LEAK_FILLER[ns * 100 : ns * 100 + 400]}" + ) + ids = tokenizer.encode(text) + tasks.append( + _async_generate( + base_url, + http, + ids, + max_new_tokens=STRESS_GEN_LEN, + ) + ) + + results = await asyncio.gather(*tasks) + for s in range(STRESS_NUM_SESSIONS): + rids[s] = results[s]["meta_info"]["id"] + + for sid in sids: + async with http.post( + base_url + "/close_session", json={"session_id": sid} + ) as resp: + assert resp.status == 200 + + +class StreamingSessionServerBase(CustomTestCase): + """Minimal streaming-session server fixture. + + Subclasses override class attrs to customize launch: + - `model`: defaults to the small model. + - `extra_args`: appended after `--enable-streaming-session` (set + `--chunked-prefill-size`, `--page-size`, spec args, etc. here). + - `env_overrides`: list of `(env_attr_name, value)` tuples; each is + pushed onto the `setUpClass` context stack so the env override is + live during `popen_launch_server` and torn down on + `tearDownClass`-time. `SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY=2` + is always applied on top of these. + """ + + model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + extra_args: list = [] + env_overrides: list = [] + + @classmethod + def setUpClass(cls): + import contextlib + + with contextlib.ExitStack() as stack: + stack.enter_context( + envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(2) + ) + for name, val in cls.env_overrides: + stack.enter_context(getattr(envs, name).override(val)) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--enable-streaming-session"] + list(cls.extra_args), + ) + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) diff --git a/scripts/ci/utils/compute_partitions.py b/scripts/ci/utils/compute_partitions.py index 1c419384f..994d877f4 100644 --- a/scripts/ci/utils/compute_partitions.py +++ b/scripts/ci/utils/compute_partitions.py @@ -45,7 +45,7 @@ _REUSABLE_STAGE_USES = "./.github/workflows/_pr-test-stage.yml" def load_run_timeouts(pr_test_yml_path: str) -> dict: - """Map `self_name -> run_timeout_minutes` from pr-test.yml. The input + """Map `self_name -> run_timeout_minutes` from one pr-test*.yml. The input is required in `_pr-test-stage.yml` -- KeyError surfaces missing. Inline stage-a-test-cpu is skipped (uses `_STAGE_A_OVERRIDES`).""" with open(pr_test_yml_path) as f: @@ -202,7 +202,7 @@ def main(): parser.add_argument( "--pr-test-yml", default=os.path.join(REPO_ROOT, ".github", "workflows", "pr-test.yml"), - help="Path to pr-test.yml; per-stage `run_timeout_minutes` is read from here.", + help="Path to pr-test*.yml; per-stage `run_timeout_minutes` is read from here.", ) args = parser.parse_args() diff --git a/test/registered/openai_server/function_call/test_tool_choice.py b/test/manual/openai_server/function_call/test_tool_choice.py similarity index 100% rename from test/registered/openai_server/function_call/test_tool_choice.py rename to test/manual/openai_server/function_call/test_tool_choice.py diff --git a/test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py b/test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py new file mode 100644 index 000000000..dc7fb9c61 --- /dev/null +++ b/test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py @@ -0,0 +1,93 @@ +"""DeepSeek-V3 FP4 4-GPU test, TRTLLM variant. + +Backend: `--attention-backend trtllm_mla --moe-runner-backend flashinfer_trtllm`. +Not registered in any CI suite -- runnable manually only. +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4" +SERVER_LAUNCH_TIMEOUT = 1200 + + +class TestDeepseekV3FP4(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--tp", + "4", + "--attention-backend", + "trtllm_mla", + "--moe-runner-backend", + "flashinfer_trtllm", + "--quantization", + "modelopt_fp4", + "--kv-cache-dtype", + "fp8_e4m3", + "--model-loader-extra-config", + '{"enable_multithread_load": true,"num_threads": 64}', + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1319, + num_threads=1319, + num_shots=8, + ) + metrics = run_eval(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n' + ) + + self.assertGreater(metrics["score"], 0.93) + + def test_bs_1_speed(self): + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + _, speed = send_one_prompt(args) + + print(f"{speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v3-fp4)\n" f"{speed=:.2f} token/s\n" + ) + + self.assertGreater(speed, 120) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/manual/spec/test_spec_ngram_fa3.py b/test/manual/spec/test_spec_ngram_fa3.py new file mode 100644 index 000000000..8c2a43dfe --- /dev/null +++ b/test/manual/spec/test_spec_ngram_fa3.py @@ -0,0 +1,18 @@ +"""NGRAM speculative-decoding test, FA3 attention-backend variant. + +Backend: `--attention-backend fa3`. +Not registered in any CI suite -- runnable manually only. +""" + +import unittest + +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.server_fixtures.ngram_fixture import NgramServerBase + + +class TestNgramSpeculativeDecodingBase(NgramServerBase, GSM8KMixin): + attention_backend = "fa3" + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/4-gpu-models/test_qwen35_hicache.py b/test/registered/4-gpu-models/test_qwen35_hicache.py index 696adfb7b..d2b478fe8 100644 --- a/test/registered/4-gpu-models/test_qwen35_hicache.py +++ b/test/registered/4-gpu-models/test_qwen35_hicache.py @@ -22,7 +22,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=540, stage="stage-c", runner_config="4-gpu-h100") +register_cuda_ci(est_time=540, stage="extra-b", runner_config="4-gpu-h100") QWEN35_27B_MODEL = "Qwen/Qwen3.5-27B" ACC_THRESHOLDS = {QWEN35_27B_MODEL: {"gsm8k": 0.8}} diff --git a/test/registered/8-gpu-models/test_deepseek_v32_indexcache.py b/test/registered/8-gpu-models/test_deepseek_v32_indexcache.py index 2809ef2c3..f710996b0 100644 --- a/test/registered/8-gpu-models/test_deepseek_v32_indexcache.py +++ b/test/registered/8-gpu-models/test_deepseek_v32_indexcache.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( write_github_step_summary, ) -register_cuda_ci(est_time=450, suite="nightly-8-gpu-h200", nightly=True) +register_cuda_ci(est_time=450, stage="extra-b", runner_config="8-gpu-h200") DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" diff --git a/test/registered/8-gpu-models/test_nvidia_nemotron_3_super_bf16.py b/test/registered/8-gpu-models/test_nvidia_nemotron_3_super_bf16.py index 5728386cc..dc395e266 100644 --- a/test/registered/8-gpu-models/test_nvidia_nemotron_3_super_bf16.py +++ b/test/registered/8-gpu-models/test_nvidia_nemotron_3_super_bf16.py @@ -11,7 +11,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=370, suite="nightly-8-gpu-h200", nightly=True) +register_cuda_ci(est_time=370, stage="extra-b", runner_config="8-gpu-h200") NEMOTRON_3_SUPER_BF16_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" diff --git a/test/registered/8-gpu-models/test_return_indexer_topk.py b/test/registered/8-gpu-models/test_return_indexer_topk.py index 7b07a5fff..2b503b9d7 100644 --- a/test/registered/8-gpu-models/test_return_indexer_topk.py +++ b/test/registered/8-gpu-models/test_return_indexer_topk.py @@ -17,7 +17,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=270, suite="nightly-8-gpu-h200", nightly=True) +register_cuda_ci(est_time=270, stage="extra-b", runner_config="8-gpu-h200") DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" diff --git a/test/registered/8-gpu-models/test_step3p5_flash_chain_mtp.py b/test/registered/8-gpu-models/test_step3p5_flash_chain_mtp.py index 357918a46..2b375888c 100644 --- a/test/registered/8-gpu-models/test_step3p5_flash_chain_mtp.py +++ b/test/registered/8-gpu-models/test_step3p5_flash_chain_mtp.py @@ -17,7 +17,7 @@ from sglang.test.test_utils import ( write_github_step_summary, ) -register_cuda_ci(est_time=480, suite="nightly-8-gpu-h200", nightly=True) +register_cuda_ci(est_time=480, stage="extra-b", runner_config="8-gpu-h200") STEP3P5_FLASH_MODEL_PATH = "stepfun-ai/Step-3.5-Flash" diff --git a/test/registered/attention/test_hybrid_attn_backend.py b/test/registered/attention/test_hybrid_attn_backend.py index 3c6ec10ca..03a15a341 100644 --- a/test/registered/attention/test_hybrid_attn_backend.py +++ b/test/registered/attention/test_hybrid_attn_backend.py @@ -1,166 +1,67 @@ import unittest -from types import SimpleNamespace -import requests - -from sglang.srt.environ import envs -from sglang.srt.utils import get_device_sm, kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.server_fixtures.hybrid_attn_backend_fixture import ( + TestHybridAttnBackendBase, +) from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE, - DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_MODEL_NAME_FOR_TEST_MLA, - DEFAULT_TARGET_MODEL_EAGLE, - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, ) # Hybrid attention backend tests (FA3 prefill + FlashInfer decode, requires SM 90+ / H100) # Multiple test classes: base, MLA, TorchCompile, SpecDecode variants -register_cuda_ci(est_time=407, stage="stage-b", runner_config="1-gpu-large") - -GSM_DATASET_PATH = None - -# Default server arguments shared across all tests -DEFAULT_SERVER_ARGS = [ - "--trust-remote-code", - "--cuda-graph-max-bs", - "8", - "--prefill-attention-backend", - "fa3", - "--decode-attention-backend", - "flashinfer", -] - - -@unittest.skipIf(get_device_sm() < 90, "Test requires CUDA SM 90 or higher") -class TestHybridAttnBackendBase(CustomTestCase): - - model = DEFAULT_MODEL_NAME_FOR_TEST - base_url = DEFAULT_URL_FOR_TEST - accuracy_threshold = 0.65 # derived tests need to override this - speculative_decode = False - spec_decode_threshold = 2.2 # derived spec decoding tests need to override this - - @classmethod - def get_server_args(cls): - """Return the arguments for the server launch. Override in subclasses.""" - return DEFAULT_SERVER_ARGS - - @classmethod - def setUpClass(cls): - # disable deep gemm precompile to make launch server faster - # please don't do this if you want to make your inference workload faster - with ( - envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.override(False), - envs.SGLANG_ENABLE_JIT_DEEPGEMM.override(False), - ): - if cls.speculative_decode: - model = DEFAULT_TARGET_MODEL_EAGLE - else: - model = cls.model - cls.process = popen_launch_server( - model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=cls.get_server_args(), - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - requests.get(self.base_url + "/flush_cache") - - model = DEFAULT_TARGET_MODEL_EAGLE if self.speculative_decode else self.model - args = SimpleNamespace( - base_url=self.base_url, - model=model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=100, - num_threads=128, - ) - metrics = run_eval(args) - print(f"{metrics=}") - - self.assertGreater(metrics["score"], self.accuracy_threshold) - - if self.speculative_decode: - server_info = requests.get(self.base_url + "/server_info") - avg_spec_accept_length = server_info.json()["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold) +register_cuda_ci(est_time=407, stage="extra-a", runner_config="1-gpu-large") class TestHybridAttnBackendMLA(TestHybridAttnBackendBase): accuracy_threshold = 0.60 model = DEFAULT_MODEL_NAME_FOR_TEST_MLA - @classmethod - def get_server_args(cls): - return DEFAULT_SERVER_ARGS - class TestHybridAttnBackendTorchCompile(TestHybridAttnBackendBase): accuracy_threshold = 0.65 - - @classmethod - def get_server_args(cls): - return DEFAULT_SERVER_ARGS + ["--enable-torch-compile"] + extra_args = ["--enable-torch-compile"] class TestHybridAttnBackendSpeculativeDecodingPrefillBackend(TestHybridAttnBackendBase): speculative_decode = True # This eagle test uses a very small model, so the accuracy is low. accuracy_threshold = 0.2 - - @classmethod - def get_server_args(cls): - return DEFAULT_SERVER_ARGS + [ - "--speculative-algorithm", - "EAGLE", - "--speculative-draft-model-path", - DEFAULT_DRAFT_MODEL_EAGLE, - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "2", - "--speculative-num-draft-tokens", - "4", - "--speculative-attention-mode", - "prefill", - ] + extra_args = [ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_DRAFT_MODEL_EAGLE, + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "2", + "--speculative-num-draft-tokens", + "4", + "--speculative-attention-mode", + "prefill", + ] class TestHybridAttnBackendSpeculativeDecodingDecodeBackend(TestHybridAttnBackendBase): speculative_decode = True # This eagle test uses a very small model, so the accuracy is low. accuracy_threshold = 0.2 - - @classmethod - def get_server_args(cls): - return DEFAULT_SERVER_ARGS + [ - "--speculative-algorithm", - "EAGLE", - "--speculative-draft-model-path", - DEFAULT_DRAFT_MODEL_EAGLE, - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "2", - "--speculative-num-draft-tokens", - "4", - "--speculative-attention-mode", - "decode", - ] + extra_args = [ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_DRAFT_MODEL_EAGLE, + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "2", + "--speculative-num-draft-tokens", + "4", + "--speculative-attention-mode", + "decode", + ] if __name__ == "__main__": diff --git a/test/registered/attention/test_torch_native_attention_backend.py b/test/registered/attention/test_torch_native_attention_backend.py index 67f76ef47..35310bd7f 100644 --- a/test/registered/attention/test_torch_native_attention_backend.py +++ b/test/registered/attention/test_torch_native_attention_backend.py @@ -18,7 +18,7 @@ from sglang.test.test_utils import ( ) # Torch native attention backend integration test with MMLU eval -register_cuda_ci(est_time=140, stage="stage-b", runner_config="1-gpu-small") +register_cuda_ci(est_time=140, stage="extra-a", runner_config="1-gpu-small") register_amd_ci(est_time=150, suite="stage-b-test-1-gpu-small-amd") diff --git a/test/registered/attention/test_triton_sliding_window.py b/test/registered/attention/test_triton_sliding_window.py index 0f03f6f11..999d6d190 100644 --- a/test/registered/attention/test_triton_sliding_window.py +++ b/test/registered/attention/test_triton_sliding_window.py @@ -16,7 +16,7 @@ from sglang.test.test_utils import ( ) # Sliding window attention with Triton backend (Gemma-3 model) -register_cuda_ci(est_time=93, stage="stage-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=93, stage="extra-a", runner_config="1-gpu-large") register_amd_ci(est_time=200, suite="stage-b-test-1-gpu-small-amd") diff --git a/test/registered/backends/test_torch_compile.py b/test/registered/backends/test_torch_compile.py index 68a9eb36f..e44a7fcba 100644 --- a/test/registered/backends/test_torch_compile.py +++ b/test/registered/backends/test_torch_compile.py @@ -15,7 +15,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=126, stage="stage-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=126, stage="extra-a", runner_config="1-gpu-large") register_amd_ci(est_time=1100, suite="stage-b-test-1-gpu-small-amd") diff --git a/test/registered/core/test_gpt_oss_sm120.py b/test/registered/core/test_gpt_oss_sm120.py index 16815bde6..862b6140e 100644 --- a/test/registered/core/test_gpt_oss_sm120.py +++ b/test/registered/core/test_gpt_oss_sm120.py @@ -5,7 +5,7 @@ import torch from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.gpt_oss_common import BaseTestGptOss -register_cuda_ci(est_time=345, stage="stage-b", runner_config="1-gpu-small") +register_cuda_ci(est_time=345, stage="extra-a", runner_config="1-gpu-small") @unittest.skipIf(not torch.cuda.is_available(), "CUDA is not available") diff --git a/test/registered/cp/test_deepseek_v32_cp_single_node.py b/test/registered/cp/test_deepseek_v32_cp_single_node.py index 85c48c8e4..fb5abcf1e 100644 --- a/test/registered/cp/test_deepseek_v32_cp_single_node.py +++ b/test/registered/cp/test_deepseek_v32_cp_single_node.py @@ -13,11 +13,7 @@ from sglang.test.test_utils import ( write_github_step_summary, ) -register_cuda_ci( - est_time=616, - stage="stage-c", - runner_config="deepep-8-gpu-h200", -) +register_cuda_ci(est_time=616, stage="extra-b", runner_config="deepep-8-gpu-h200") DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" diff --git a/test/registered/distributed/test_disaggregation_hybrid_attention.py b/test/registered/distributed/test_disaggregation_hybrid_attention.py index faf388606..240f6506e 100644 --- a/test/registered/distributed/test_disaggregation_hybrid_attention.py +++ b/test/registered/distributed/test_disaggregation_hybrid_attention.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( popen_launch_pd_server, ) -register_cuda_ci(est_time=310, suite="nightly-8-gpu-h200", nightly=True) +register_cuda_ci(est_time=310, stage="extra-b", runner_config="8-gpu-h200") @unittest.skipIf(is_in_ci(), "Temporarily disable the flaky test.") diff --git a/test/registered/distributed/test_load_weights_from_remote_instance.py b/test/registered/distributed/test_load_weights_from_remote_instance.py index 50a659244..42db54686 100644 --- a/test/registered/distributed/test_load_weights_from_remote_instance.py +++ b/test/registered/distributed/test_load_weights_from_remote_instance.py @@ -38,7 +38,7 @@ from sglang.utils import terminate_process mp.set_start_method("spawn", force=True) -register_cuda_ci(est_time=145, stage="stage-b", runner_config="2-gpu-large") +register_cuda_ci(est_time=145, stage="extra-a", runner_config="2-gpu-large") register_amd_ci(est_time=72, suite="stage-b-test-2-gpu-large-amd") diff --git a/test/registered/ep/test_deepep_large.py b/test/registered/ep/test_deepep_large.py index b032dc4ab..c9c183c3d 100644 --- a/test/registered/ep/test_deepep_large.py +++ b/test/registered/ep/test_deepep_large.py @@ -16,7 +16,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=528, stage="stage-c", runner_config="deepep-8-gpu-h200") +register_cuda_ci(est_time=528, stage="extra-b", runner_config="deepep-8-gpu-h200") DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" diff --git a/test/registered/hicache/test_hicache_spec_file_storage.py b/test/registered/hicache/test_hicache_spec_file_storage.py index 602530764..4ea14c26e 100644 --- a/test/registered/hicache/test_hicache_spec_file_storage.py +++ b/test/registered/hicache/test_hicache_spec_file_storage.py @@ -30,7 +30,7 @@ from sglang.test.test_utils import ( ) from sglang.utils import wait_for_http_ready -register_cuda_ci(est_time=200, stage="stage-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=200, stage="extra-a", runner_config="1-gpu-large") @unittest.skipIf(is_hip(), "HiCache + EAGLE3 file-storage loadback e2e is CUDA-only.") diff --git a/test/registered/lora/test_lora_drainer.py b/test/registered/lora/test_lora_drainer.py index e1b68fa9d..2aa385b64 100644 --- a/test/registered/lora/test_lora_drainer.py +++ b/test/registered/lora/test_lora_drainer.py @@ -12,7 +12,7 @@ from sglang.test.lora_utils import ( ) from sglang.test.test_utils import is_in_ci -register_cuda_ci(est_time=100, stage="stage-b", runner_config="1-gpu-small") +register_cuda_ci(est_time=100, stage="extra-a", runner_config="1-gpu-small") register_amd_ci(est_time=100, suite="stage-b-test-1-gpu-small-amd") MOCK_START_TIME = 1000.0 diff --git a/test/registered/lora/test_lora_eviction.py b/test/registered/lora/test_lora_eviction.py index 6fad72acf..c01310abf 100644 --- a/test/registered/lora/test_lora_eviction.py +++ b/test/registered/lora/test_lora_eviction.py @@ -23,7 +23,7 @@ from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.runners import SRTRunner from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=263, stage="stage-b", runner_config="1-gpu-small") +register_cuda_ci(est_time=263, stage="extra-a", runner_config="1-gpu-small") register_amd_ci(est_time=224, suite="stage-b-test-1-gpu-small-amd") PROMPTS = [ diff --git a/test/registered/lora/test_lora_gpt_oss_20b_logprob_diff.py b/test/registered/lora/test_lora_gpt_oss_20b_logprob_diff.py index a57b8ffcd..4482dab05 100644 --- a/test/registered/lora/test_lora_gpt_oss_20b_logprob_diff.py +++ b/test/registered/lora/test_lora_gpt_oss_20b_logprob_diff.py @@ -34,11 +34,7 @@ import sglang as sgl from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci( - est_time=90, - suite="nightly-4-gpu-b200", - nightly=True, -) +register_cuda_ci(est_time=90, stage="extra-b", runner_config="4-gpu-b200") BASE_MODEL = "lmsys/gpt-oss-20b-bf16" LORA_HF_REPO = "yushengsu/lora-diff-gpt-oss-20b" diff --git a/test/registered/lora/test_lora_moe_tp_logprob_diff.py b/test/registered/lora/test_lora_moe_tp_logprob_diff.py index 9214a9e1e..cd277662b 100644 --- a/test/registered/lora/test_lora_moe_tp_logprob_diff.py +++ b/test/registered/lora/test_lora_moe_tp_logprob_diff.py @@ -32,11 +32,7 @@ from sglang.test.test_utils import ( is_in_ci, ) -register_cuda_ci( - est_time=200, - stage="stage-b", - runner_config="2-gpu-large", -) +register_cuda_ci(est_time=200, stage="extra-a", runner_config="2-gpu-large") LOGPROB_THRESHOLD = 5e-04 MAX_NEW_TOKENS = 10 diff --git a/test/registered/lora/test_lora_nemotron_3_super_120b_a12b_logprob_diff.py b/test/registered/lora/test_lora_nemotron_3_super_120b_a12b_logprob_diff.py index 41a4c47fb..fcdc0bf0c 100644 --- a/test/registered/lora/test_lora_nemotron_3_super_120b_a12b_logprob_diff.py +++ b/test/registered/lora/test_lora_nemotron_3_super_120b_a12b_logprob_diff.py @@ -34,11 +34,7 @@ import sglang as sgl from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci( - est_time=100, - suite="nightly-4-gpu-b200", - nightly=True, -) +register_cuda_ci(est_time=100, stage="extra-b", runner_config="4-gpu-b200") BASE_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" LORA_HF_REPO = "opherlie/lora-test-case-NVIDIA-Nemotron-3-Super-120B-A12B-BF16" diff --git a/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py b/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py index 3f5a7e55c..58e9ffb87 100644 --- a/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py +++ b/test/registered/lora/test_lora_qwen3_30b_a3b_instruct_2507_logprob_diff.py @@ -34,11 +34,7 @@ import sglang as sgl from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci( - est_time=100, - suite="nightly-4-gpu-b200", - nightly=True, -) +register_cuda_ci(est_time=100, stage="extra-b", runner_config="4-gpu-b200") BASE_MODEL = "Qwen/Qwen3-30B-A3B-Instruct-2507" LORA_HF_REPO = "yushengsu/lora-diff-Qwen3-30B-A3B-Instruct-2507" diff --git a/test/registered/lora/test_lora_qwen3_5_4b_logprob_diff.py b/test/registered/lora/test_lora_qwen3_5_4b_logprob_diff.py index 8c514da48..912222039 100644 --- a/test/registered/lora/test_lora_qwen3_5_4b_logprob_diff.py +++ b/test/registered/lora/test_lora_qwen3_5_4b_logprob_diff.py @@ -34,11 +34,7 @@ import sglang as sgl from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci( - est_time=90, - stage="stage-b", - runner_config="1-gpu-large", -) +register_cuda_ci(est_time=90, stage="extra-a", runner_config="1-gpu-large") BASE_MODEL = "Qwen/Qwen3.5-4B" LORA_HF_REPO = "opherlie/lora-test-case-Qwen3.5-4B" diff --git a/test/registered/lora/test_lora_qwen3_8b_logprob_diff.py b/test/registered/lora/test_lora_qwen3_8b_logprob_diff.py index 7f2f9a431..a11386c34 100644 --- a/test/registered/lora/test_lora_qwen3_8b_logprob_diff.py +++ b/test/registered/lora/test_lora_qwen3_8b_logprob_diff.py @@ -37,11 +37,7 @@ from sglang.srt.lora.utils import auto_detect_lora_target_modules from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci( - est_time=40, - stage="stage-b", - runner_config="1-gpu-large", -) +register_cuda_ci(est_time=40, stage="extra-a", runner_config="1-gpu-large") BASE_MODEL = "Qwen/Qwen3-8B" LORA_HF_REPO = "yushengsu/lora-diff-Qwen3-8B" diff --git a/test/registered/models/test_compressed_tensors_models.py b/test/registered/models/test_compressed_tensors_models.py index 2d4d160e5..6644a268b 100644 --- a/test/registered/models/test_compressed_tensors_models.py +++ b/test/registered/models/test_compressed_tensors_models.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=65, stage="stage-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=65, stage="extra-a", runner_config="1-gpu-large") register_amd_ci(est_time=42, suite="stage-b-test-1-gpu-small-amd") diff --git a/test/registered/models/test_generation_models.py b/test/registered/models/test_generation_models.py index b6b2ec1c5..4c1bd79b3 100644 --- a/test/registered/models/test_generation_models.py +++ b/test/registered/models/test_generation_models.py @@ -1,7 +1,7 @@ from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci # Generation model tests (CUDA only) -register_cuda_ci(est_time=150, stage="stage-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=150, stage="extra-a", runner_config="1-gpu-large") register_amd_ci(est_time=106, suite="stage-b-test-1-gpu-small-amd") # Copyright 2023-2024 SGLang Team diff --git a/test/registered/models/test_ministral4_models.py b/test/registered/models/test_ministral4_models.py index 9697068c1..2f6656e1d 100644 --- a/test/registered/models/test_ministral4_models.py +++ b/test/registered/models/test_ministral4_models.py @@ -6,11 +6,7 @@ from sglang.test.kits.mmmu_vlm_kit import MMMUMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase -register_cuda_ci( - est_time=200, - stage="stage-b", - runner_config="2-gpu-large", -) +register_cuda_ci(est_time=200, stage="extra-a", runner_config="2-gpu-large") MODEL = "mistralai/Mistral-Small-4-119B-2603" diff --git a/test/registered/models/test_vlm_models.py b/test/registered/models/test_vlm_models.py index e7da07f6b..e6dd05e2f 100644 --- a/test/registered/models/test_vlm_models.py +++ b/test/registered/models/test_vlm_models.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import is_in_ci # VLM (Vision Language Model) tests -register_cuda_ci(est_time=317, stage="stage-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=317, stage="extra-a", runner_config="1-gpu-large") register_amd_ci(est_time=850, suite="stage-b-test-1-gpu-small-amd-nondeterministic") _is_hip = is_hip() diff --git a/test/registered/moe/test_cutedsl_moe.py b/test/registered/moe/test_cutedsl_moe.py index 3ef21a261..ed54c206d 100644 --- a/test/registered/moe/test_cutedsl_moe.py +++ b/test/registered/moe/test_cutedsl_moe.py @@ -16,7 +16,7 @@ except ImportError: CuteDslMoEWrapper = None convert_sf_to_mma_layout = None -register_cuda_ci(est_time=24, suite="nightly-4-gpu-b200", nightly=True) +register_cuda_ci(est_time=24, stage="extra-b", runner_config="4-gpu-b200") SKIP_TEST = torch.cuda.get_device_capability() < (10, 0) SKIP_REASON = "Nvfp4 Requires compute capability of 10 or above." diff --git a/test/registered/moe/test_moe_ep.py b/test/registered/moe/test_moe_ep.py index e4681fb7a..cd1120603 100644 --- a/test/registered/moe/test_moe_ep.py +++ b/test/registered/moe/test_moe_ep.py @@ -12,6 +12,8 @@ from sglang.test.test_utils import ( popen_launch_server, ) +# Per-commit: TP=2 EP=2 baseline. +# DeepGEMM/FP8 variant moved to test_moe_ep_nightly.py. register_cuda_ci(est_time=279, stage="stage-b", runner_config="2-gpu-large") @@ -52,46 +54,5 @@ class TestEp(CustomTestCase): self.assertGreater(metrics["score"], 0.60) -class TestEpDeepGEMM(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--trust-remote-code", - "--tp", - "2", - "--ep-size", - "2", - "--quantization", - "fp8", - "--moe-runner-backend", - "deep_gemm", - ], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - args = SimpleNamespace( - base_url=self.base_url, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=200, - num_threads=128, - ) - metrics = run_eval(args) - print(metrics) - - self.assertGreater(metrics["score"], 0.60) - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/moe/test_moe_ep_extra.py b/test/registered/moe/test_moe_ep_extra.py new file mode 100644 index 000000000..1047b1708 --- /dev/null +++ b/test/registered/moe/test_moe_ep_extra.py @@ -0,0 +1,66 @@ +"""Extra: TP=2 EP=2 with FP8 + DeepGEMM MoE backend. + +Sibling per-commit file (test_moe_ep.py) keeps the baseline TP=2 EP=2 +variant. +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +register_cuda_ci(est_time=279, stage="extra-a", runner_config="2-gpu-large") + + +class TestEpDeepGEMM(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--ep-size", + "2", + "--quantization", + "fp8", + "--moe-runner-backend", + "deep_gemm", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + base_url=self.base_url, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, + ) + metrics = run_eval(args) + print(metrics) + + self.assertGreater(metrics["score"], 0.60) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/observability/test_tracing.py b/test/registered/observability/test_tracing.py index ca450c33d..7d132ceff 100644 --- a/test/registered/observability/test_tracing.py +++ b/test/registered/observability/test_tracing.py @@ -46,7 +46,7 @@ from sglang.test.test_utils import ( logger = logging.getLogger(__name__) # CI registration -register_cuda_ci(est_time=113, stage="stage-b", runner_config="1-gpu-small") +register_cuda_ci(est_time=113, stage="extra-a", runner_config="1-gpu-small") # ============================================================================ diff --git a/test/registered/perf/test_bench_one_batch_2gpu.py b/test/registered/perf/test_bench_one_batch_2gpu.py index bb079075d..ce61984e7 100644 --- a/test/registered/perf/test_bench_one_batch_2gpu.py +++ b/test/registered/perf/test_bench_one_batch_2gpu.py @@ -11,7 +11,7 @@ from sglang.test.test_utils import ( write_github_step_summary, ) -register_cuda_ci(est_time=209, stage="stage-b", runner_config="2-gpu-large") +register_cuda_ci(est_time=209, stage="extra-a", runner_config="2-gpu-large") register_amd_ci(est_time=630, suite="stage-b-test-2-gpu-large-amd") diff --git a/test/registered/perf/test_bench_serving_1gpu_large.py b/test/registered/perf/test_bench_serving_1gpu_large.py index 134c8e3ea..e178f58bf 100644 --- a/test/registered/perf/test_bench_serving_1gpu_large.py +++ b/test/registered/perf/test_bench_serving_1gpu_large.py @@ -17,7 +17,7 @@ from sglang.test.test_utils import ( write_github_step_summary, ) -register_cuda_ci(est_time=286, stage="stage-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=286, stage="extra-a", runner_config="1-gpu-large") register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-large-amd") diff --git a/test/registered/perf/test_bench_serving_1gpu_part1.py b/test/registered/perf/test_bench_serving_1gpu_part1.py index 85238e00b..ef257e4b7 100644 --- a/test/registered/perf/test_bench_serving_1gpu_part1.py +++ b/test/registered/perf/test_bench_serving_1gpu_part1.py @@ -19,7 +19,7 @@ from sglang.test.test_utils import ( write_github_step_summary, ) -register_cuda_ci(est_time=1210, stage="stage-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=1210, stage="extra-a", runner_config="1-gpu-large") register_amd_ci(est_time=1100, suite="stage-b-test-1-gpu-large-amd") diff --git a/test/registered/perf/test_bench_serving_1gpu_part2.py b/test/registered/perf/test_bench_serving_1gpu_part2.py index dc5cf99fc..4f9cc12f5 100644 --- a/test/registered/perf/test_bench_serving_1gpu_part2.py +++ b/test/registered/perf/test_bench_serving_1gpu_part2.py @@ -19,7 +19,7 @@ from sglang.test.test_utils import ( write_github_step_summary, ) -register_cuda_ci(est_time=968, stage="stage-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=968, stage="extra-a", runner_config="1-gpu-large") register_amd_ci(est_time=900, suite="stage-b-test-1-gpu-large-amd") diff --git a/test/registered/perf/test_bench_serving_2gpu.py b/test/registered/perf/test_bench_serving_2gpu.py index 66d2bc104..56712d369 100644 --- a/test/registered/perf/test_bench_serving_2gpu.py +++ b/test/registered/perf/test_bench_serving_2gpu.py @@ -14,7 +14,7 @@ from sglang.test.test_utils import ( write_github_step_summary, ) -register_cuda_ci(est_time=721, stage="stage-b", runner_config="2-gpu-large") +register_cuda_ci(est_time=721, stage="extra-a", runner_config="2-gpu-large") register_amd_ci(est_time=1450, suite="stage-b-test-2-gpu-large-amd") diff --git a/test/registered/piecewise_cuda_graph/test_pcg_with_speculative_decoding.py b/test/registered/piecewise_cuda_graph/test_pcg_with_speculative_decoding.py index 0386babb2..3bd1ef0da 100644 --- a/test/registered/piecewise_cuda_graph/test_pcg_with_speculative_decoding.py +++ b/test/registered/piecewise_cuda_graph/test_pcg_with_speculative_decoding.py @@ -1,242 +1,44 @@ -"""Test piecewise CUDA graph coexisting with speculative decoding. +"""Test piecewise CUDA graph coexisting with speculative decoding (EAGLE3). -PCG handles prefill/extend path while speculative decoding (MTP/EAGLE3/STANDALONE/NGRAM) -uses decode CUDA graphs. This test verifies they don't interfere with each other. +PCG handles prefill/extend path while speculative decoding (EAGLE3) uses +decode CUDA graphs. This test verifies they don't interfere with each +other. MTP / STANDALONE / NGRAM variants moved to the sibling file +test_pcg_with_speculative_decoding_extra.py. """ import unittest -from types import SimpleNamespace -import requests - -from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - popen_launch_server, -) +from sglang.test.server_fixtures.pcg_spec_fixture import PCGSpecBase register_cuda_ci(est_time=531, stage="stage-b", runner_config="2-gpu-large") -class TestPCGWithMTP(unittest.TestCase): - """Test PCG + MTP (NEXTN) on Qwen3.5-35B-A3B with FP8.""" +class TestPCGWithEAGLE3(PCGSpecBase, unittest.TestCase): + """PCG + EAGLE3 on Qwen3-30B-A3B-Instruct-2507.""" - @classmethod - def setUpClass(cls): - cls.model = "Qwen/Qwen3.5-35B-A3B" - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--tp", - "2", - "--trust-remote-code", - "--quantization", - "fp8", - "--mamba-scheduler-strategy", - "extra_buffer", - "--enable-piecewise-cuda-graph", - "--speculative-algorithm", - "NEXTN", - "--reasoning-parser", - "qwen3", - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - max_tokens=8192, - num_examples=200, - num_threads=200, - thinking_mode="qwen3", - ) - metrics = run_eval(args) - print(metrics) - self.assertGreater(metrics["score"], 0.75) - - server_info = requests.get(self.base_url + "/server_info").json() - avg_spec_accept_length = server_info["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - self.assertGreater(avg_spec_accept_length, 1.5) - - -class TestPCGWithEAGLE3(unittest.TestCase): - """Test PCG + EAGLE3 on Qwen3-30B-A3B-Instruct-2507.""" - - @classmethod - def setUpClass(cls): - cls.model = "Qwen/Qwen3-30B-A3B-Instruct-2507" - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--tp", - "2", - "--trust-remote-code", - "--enforce-piecewise-cuda-graph", - "--mem-fraction-static", - "0.6", - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model-path", - "lmsys/SGLang-EAGLE3-Qwen3-30B-A3B-Instruct-2507-SpecForge-Nex", - "--speculative-num-steps", - "5", - "--speculative-eagle-topk", - "4", - "--speculative-num-draft-tokens", - "8", - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3, - other_args=other_args, - env={"SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN": "1"}, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - max_tokens=512, - num_examples=200, - num_threads=200, - ) - metrics = run_eval(args) - print(metrics) - self.assertGreater(metrics["score"], 0.75) - - server_info = requests.get(self.base_url + "/server_info").json() - avg_spec_accept_length = server_info["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - self.assertGreater(avg_spec_accept_length, 1.5) - - -class TestPCGWithSTANDALONE(unittest.TestCase): - """Test PCG + STANDALONE on Llama-3.1-8B-Instruct + Llama-3.2-1B-Instruct.""" - - @classmethod - def setUpClass(cls): - cls.model = "meta-llama/Llama-3.1-8B-Instruct" - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--trust-remote-code", - "--enforce-piecewise-cuda-graph", - "--mem-fraction-static", - "0.5", - "--speculative-algorithm", - "STANDALONE", - "--speculative-draft-model-path", - "meta-llama/Llama-3.2-1B-Instruct", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 2, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - max_tokens=512, - num_examples=200, - num_threads=200, - ) - metrics = run_eval(args) - print(metrics) - self.assertGreater(metrics["score"], 0.50) - - server_info = requests.get(self.base_url + "/server_info").json() - avg_spec_accept_length = server_info["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - self.assertGreater(avg_spec_accept_length, 1.5) - - -class TestPCGWithNGRAM(unittest.TestCase): - """Test PCG + NGRAM on Qwen2.5-Coder-7B-Instruct.""" - - @classmethod - def setUpClass(cls): - cls.model = "Qwen/Qwen2.5-Coder-7B-Instruct" - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--trust-remote-code", - "--enforce-piecewise-cuda-graph", - "--speculative-algorithm", - "NGRAM", - "--speculative-num-draft-tokens", - "16", - "--cuda-graph-max-bs", - "8", - "--mem-fraction-static", - "0.8", - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 2, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - max_tokens=512, - num_examples=200, - num_threads=200, - ) - metrics = run_eval(args) - print(metrics) - self.assertGreater(metrics["score"], 0.70) - - server_info = requests.get(self.base_url + "/server_info").json() - avg_spec_accept_length = server_info["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - self.assertGreater(avg_spec_accept_length, 1.5) + model = "Qwen/Qwen3-30B-A3B-Instruct-2507" + server_args = [ + "--tp", + "2", + "--trust-remote-code", + "--enforce-piecewise-cuda-graph", + "--mem-fraction-static", + "0.6", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + "lmsys/SGLang-EAGLE3-Qwen3-30B-A3B-Instruct-2507-SpecForge-Nex", + "--speculative-num-steps", + "5", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "8", + ] + timeout_mult = 3 + server_env = {"SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN": "1"} + accuracy_threshold = 0.75 if __name__ == "__main__": diff --git a/test/registered/piecewise_cuda_graph/test_pcg_with_speculative_decoding_extra.py b/test/registered/piecewise_cuda_graph/test_pcg_with_speculative_decoding_extra.py new file mode 100644 index 000000000..70fadf6e9 --- /dev/null +++ b/test/registered/piecewise_cuda_graph/test_pcg_with_speculative_decoding_extra.py @@ -0,0 +1,81 @@ +"""Extra: PCG coexistence with non-EAGLE3 speculative decoding variants. + +EAGLE3 stays per-commit in the sibling file +test_pcg_with_speculative_decoding.py. +""" + +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.server_fixtures.pcg_spec_fixture import PCGSpecBase + +register_cuda_ci(est_time=531, stage="extra-a", runner_config="2-gpu-large") + + +class TestPCGWithMTP(PCGSpecBase, unittest.TestCase): + """PCG + MTP (NEXTN) on Qwen3.5-35B-A3B with FP8.""" + + model = "Qwen/Qwen3.5-35B-A3B" + server_args = [ + "--tp", + "2", + "--trust-remote-code", + "--quantization", + "fp8", + "--mamba-scheduler-strategy", + "extra_buffer", + "--enable-piecewise-cuda-graph", + "--speculative-algorithm", + "NEXTN", + "--reasoning-parser", + "qwen3", + ] + timeout_mult = 3 + max_tokens = 8192 + thinking_mode = "qwen3" + accuracy_threshold = 0.75 + + +class TestPCGWithSTANDALONE(PCGSpecBase, unittest.TestCase): + """PCG + STANDALONE on Llama-3.1-8B-Instruct + Llama-3.2-1B-Instruct.""" + + model = "meta-llama/Llama-3.1-8B-Instruct" + server_args = [ + "--trust-remote-code", + "--enforce-piecewise-cuda-graph", + "--mem-fraction-static", + "0.5", + "--speculative-algorithm", + "STANDALONE", + "--speculative-draft-model-path", + "meta-llama/Llama-3.2-1B-Instruct", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + ] + accuracy_threshold = 0.50 + + +class TestPCGWithNGRAM(PCGSpecBase, unittest.TestCase): + """PCG + NGRAM on Qwen2.5-Coder-7B-Instruct.""" + + model = "Qwen/Qwen2.5-Coder-7B-Instruct" + server_args = [ + "--trust-remote-code", + "--enforce-piecewise-cuda-graph", + "--speculative-algorithm", + "NGRAM", + "--speculative-num-draft-tokens", + "16", + "--cuda-graph-max-bs", + "8", + "--mem-fraction-static", + "0.8", + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/quant/test_deepseek_v3_fp4_4gpu.py b/test/registered/quant/test_deepseek_v3_fp4_4gpu.py index fd716f862..5eadc8287 100644 --- a/test/registered/quant/test_deepseek_v3_fp4_4gpu.py +++ b/test/registered/quant/test_deepseek_v3_fp4_4gpu.py @@ -1,11 +1,9 @@ -import os import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_eval import run_eval -from sglang.test.send_one import BenchArgs, send_one_prompt from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, @@ -14,137 +12,15 @@ from sglang.test.test_utils import ( write_github_step_summary, ) +# Per-commit: SymmetricMemory variant only. +# - TestDeepseekV3FP4 (TRTLLM) archived to test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py +# - TestDeepseekV3FP4CutlassMoE moved to test_deepseek_v3_fp4_4gpu_extra.py register_cuda_ci(est_time=960, stage="stage-c", runner_config="4-gpu-b200") FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4" SERVER_LAUNCH_TIMEOUT = 1200 -class TestDeepseekV3FP4(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--tp", - "4", - "--attention-backend", - "trtllm_mla", - "--moe-runner-backend", - "flashinfer_trtllm", - "--quantization", - "modelopt_fp4", - "--kv-cache-dtype", - "fp8_e4m3", - "--model-loader-extra-config", - '{"enable_multithread_load": true,"num_threads": 64}', - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=SERVER_LAUNCH_TIMEOUT, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_a_gsm8k( - self, - ): # Append an "a" to make this test run first (alphabetically) to warm up the server - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=1319, - num_threads=1319, - num_shots=8, - ) - metrics = run_eval(args) - print(f"{metrics=}") - - if is_in_ci(): - write_github_step_summary( - f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n' - ) - - self.assertGreater(metrics["score"], 0.93) - - def test_bs_1_speed(self): - args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) - _, speed = send_one_prompt(args) - - print(f"{speed=:.2f}") - - if is_in_ci(): - write_github_step_summary( - f"### test_bs_1_speed (deepseek-v3-fp4)\n" f"{speed=:.2f} token/s\n" - ) - - self.assertGreater(speed, 120) - - -class TestDeepseekV3FP4CutlassMoE(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--tp", - "4", - "--ep", - "4", - "--attention-backend", - "trtllm_mla", - "--moe-runner-backend", - "flashinfer_cutlass", - "--quantization", - "modelopt_fp4", - "--model-loader-extra-config", - '{"enable_multithread_load": true}', - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=SERVER_LAUNCH_TIMEOUT, - other_args=other_args, - env={ - **os.environ, - "SGLANG_MOE_NVFP4_DISPATCH": "1", # Enable nvfp4 all gather - }, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_a_gsm8k( - self, - ): # Append an "a" to make this test run first (alphabetically) to warm up the server - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=1319, - num_threads=1319, - num_shots=8, - ) - metrics = run_eval(args) - print(f"{metrics=}") - - if is_in_ci(): - write_github_step_summary( - f"### test_gsm8k (deepseek-v3-fp4-cutlass-moe)\n" - f'{metrics["score"]=:.3f}\n' - ) - self.assertGreater(metrics["score"], 0.93) - - class TestDeepseekV3FP4SymmetricMemory(CustomTestCase): @classmethod def setUpClass(cls): diff --git a/test/registered/quant/test_deepseek_v3_fp4_4gpu_extra.py b/test/registered/quant/test_deepseek_v3_fp4_4gpu_extra.py new file mode 100644 index 000000000..d351e7fcc --- /dev/null +++ b/test/registered/quant/test_deepseek_v3_fp4_4gpu_extra.py @@ -0,0 +1,87 @@ +"""Extra: DeepSeek-V3 FP4 with FlashInfer Cutlass MoE backend. + +Sibling per-commit file (test_deepseek_v3_fp4_4gpu.py) keeps the +SymmetricMemory variant. +""" + +import os +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +register_cuda_ci(est_time=960, stage="extra-b", runner_config="4-gpu-b200") + +FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4" +SERVER_LAUNCH_TIMEOUT = 1200 + + +class TestDeepseekV3FP4CutlassMoE(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--tp", + "4", + "--ep", + "4", + "--attention-backend", + "trtllm_mla", + "--moe-runner-backend", + "flashinfer_cutlass", + "--quantization", + "modelopt_fp4", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + env={ + **os.environ, + "SGLANG_MOE_NVFP4_DISPATCH": "1", # Enable nvfp4 all gather + }, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1319, + num_threads=1319, + num_shots=8, + ) + metrics = run_eval(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v3-fp4-cutlass-moe)\n" + f'{metrics["score"]=:.3f}\n' + ) + self.assertGreater(metrics["score"], 0.93) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/quant/test_fp8_blockwise_gemm.py b/test/registered/quant/test_fp8_blockwise_gemm.py index a7257fd21..3e31c1f2a 100644 --- a/test/registered/quant/test_fp8_blockwise_gemm.py +++ b/test/registered/quant/test_fp8_blockwise_gemm.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( try_cached_model, ) -register_cuda_ci(est_time=430, suite="nightly-4-gpu-b200", nightly=True) +register_cuda_ci(est_time=430, stage="extra-b", runner_config="4-gpu-b200") MODEL_PATH = "Qwen/Qwen3-4B-Instruct-2507-FP8" MXFP8_MODEL_PATH = "zianglih/Qwen3-4B-Instruct-2507-MXFP8" diff --git a/test/registered/quant/test_fp8_gemm_sm120.py b/test/registered/quant/test_fp8_gemm_sm120.py index 3817f298d..143df6a4a 100644 --- a/test/registered/quant/test_fp8_gemm_sm120.py +++ b/test/registered/quant/test_fp8_gemm_sm120.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( try_cached_model, ) -register_cuda_ci(est_time=146, stage="stage-b", runner_config="1-gpu-small") +register_cuda_ci(est_time=146, stage="extra-a", runner_config="1-gpu-small") PERTENSOR_MODEL_PATH = "nvidia/Llama-3.1-8B-Instruct-FP8" BLOCKWISE_MODEL_PATH = "Qwen/Qwen3-4B-Instruct-2507-FP8" diff --git a/test/registered/quant/test_fp8kv_triton.py b/test/registered/quant/test_fp8kv_triton.py index 93103e143..e9b867538 100644 --- a/test/registered/quant/test_fp8kv_triton.py +++ b/test/registered/quant/test_fp8kv_triton.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=73, stage="stage-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=73, stage="extra-a", runner_config="1-gpu-large") class TestFP8KVCacheTritonBackend(CustomTestCase): diff --git a/test/registered/quant/test_gptqmodel_dynamic.py b/test/registered/quant/test_gptqmodel_dynamic.py index ca9b12347..47e12d0ad 100644 --- a/test/registered/quant/test_gptqmodel_dynamic.py +++ b/test/registered/quant/test_gptqmodel_dynamic.py @@ -14,7 +14,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=100, stage="stage-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=100, stage="extra-a", runner_config="1-gpu-large") def check_quant_method(model_path: str, use_marlin_kernel: bool): diff --git a/test/registered/quant/test_w8a8_quantization.py b/test/registered/quant/test_w8a8_quantization.py index c4ff01215..02160bc92 100644 --- a/test/registered/quant/test_w8a8_quantization.py +++ b/test/registered/quant/test_w8a8_quantization.py @@ -14,7 +14,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=232, stage="stage-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=232, stage="extra-a", runner_config="1-gpu-large") class BaseW8A8Test(CustomTestCase): diff --git a/test/registered/rl/test_lora_load_from_tensor.py b/test/registered/rl/test_lora_load_from_tensor.py index a0e7f9655..d2505215a 100644 --- a/test/registered/rl/test_lora_load_from_tensor.py +++ b/test/registered/rl/test_lora_load_from_tensor.py @@ -10,7 +10,7 @@ import sglang as sgl from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=102, stage="stage-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=102, stage="extra-a", runner_config="1-gpu-large") register_amd_ci(est_time=90, suite="stage-b-test-1-gpu-small-amd") MODEL_PATH = "Qwen/Qwen3-0.6B" diff --git a/test/registered/rl/test_multi_instance_release_memory_occupation.py b/test/registered/rl/test_multi_instance_release_memory_occupation.py index 754e3532d..610e77c7b 100644 --- a/test/registered/rl/test_multi_instance_release_memory_occupation.py +++ b/test/registered/rl/test_multi_instance_release_memory_occupation.py @@ -20,7 +20,7 @@ from sglang.test.test_utils import ( find_available_port, ) -register_cuda_ci(est_time=57, stage="stage-c", runner_config="4-gpu-h100") +register_cuda_ci(est_time=57, stage="extra-b", runner_config="4-gpu-h100") register_amd_ci( est_time=64, suite="stage-c-test-4-gpu-amd", diff --git a/test/registered/rl/test_return_routed_experts.py b/test/registered/rl/test_return_routed_experts.py index 14fd3ce22..5157616ef 100644 --- a/test/registered/rl/test_return_routed_experts.py +++ b/test/registered/rl/test_return_routed_experts.py @@ -24,7 +24,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=400, stage="stage-c", runner_config="4-gpu-h100") +register_cuda_ci(est_time=400, stage="extra-b", runner_config="4-gpu-h100") # FP8 variant of Qwen3-30B-A3B: required because DeepEP normal/LL fast paths in # ep_moe/layer.py only run for {Fp8Config (via deep_gemm), W4AFp8Config, aiter, diff --git a/test/registered/rl/test_update_weights_from_disk_blackwell.py b/test/registered/rl/test_update_weights_from_disk_blackwell.py index 00ee93076..086488965 100644 --- a/test/registered/rl/test_update_weights_from_disk_blackwell.py +++ b/test/registered/rl/test_update_weights_from_disk_blackwell.py @@ -1,6 +1,6 @@ from sglang.test.ci.ci_register import register_cuda_ci -register_cuda_ci(est_time=320, suite="nightly-4-gpu-b200", nightly=True) +register_cuda_ci(est_time=320, stage="extra-b", runner_config="4-gpu-b200") import unittest diff --git a/test/registered/rl/test_update_weights_from_distributed.py b/test/registered/rl/test_update_weights_from_distributed.py index 751348c76..fe7e593a3 100644 --- a/test/registered/rl/test_update_weights_from_distributed.py +++ b/test/registered/rl/test_update_weights_from_distributed.py @@ -43,7 +43,7 @@ from sglang.test.test_utils import ( ) from sglang.utils import terminate_process -register_cuda_ci(est_time=137, stage="stage-b", runner_config="2-gpu-large") +register_cuda_ci(est_time=137, stage="extra-a", runner_config="2-gpu-large") register_amd_ci(est_time=400, suite="stage-b-test-2-gpu-large-amd") mp.set_start_method("spawn", force=True) diff --git a/test/registered/rl/test_update_weights_from_tensor.py b/test/registered/rl/test_update_weights_from_tensor.py index c5115d4e8..086d32950 100644 --- a/test/registered/rl/test_update_weights_from_tensor.py +++ b/test/registered/rl/test_update_weights_from_tensor.py @@ -1,6 +1,6 @@ from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -register_cuda_ci(est_time=147, stage="stage-b", runner_config="1-gpu-small") +register_cuda_ci(est_time=147, stage="extra-a", runner_config="1-gpu-small") register_amd_ci(est_time=195, suite="stage-b-test-1-gpu-small-amd") import gc diff --git a/test/registered/scheduler/test_priority_scheduling.py b/test/registered/scheduler/test_priority_scheduling.py index 1d4eba278..5175f5bdb 100644 --- a/test/registered/scheduler/test_priority_scheduling.py +++ b/test/registered/scheduler/test_priority_scheduling.py @@ -17,7 +17,7 @@ from sglang.test.test_utils import ( send_concurrent_generate_requests_with_custom_params, ) -register_cuda_ci(est_time=149, stage="stage-b", runner_config="1-gpu-small") +register_cuda_ci(est_time=149, stage="extra-a", runner_config="1-gpu-small") register_amd_ci(est_time=195, suite="stage-b-test-1-gpu-small-amd") diff --git a/test/registered/sessions/test_session_control.py b/test/registered/sessions/test_session_control.py index 1e86974d0..a5ff0b847 100644 --- a/test/registered/sessions/test_session_control.py +++ b/test/registered/sessions/test_session_control.py @@ -24,7 +24,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=87, stage="stage-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=87, stage="extra-a", runner_config="1-gpu-large") def remove_prefix(text: str, prefix: str) -> str: diff --git a/test/registered/sessions/test_session_latency.py b/test/registered/sessions/test_session_latency.py index c87529f13..c0f79e9d4 100644 --- a/test/registered/sessions/test_session_latency.py +++ b/test/registered/sessions/test_session_latency.py @@ -31,11 +31,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci( - est_time=122, - stage="stage-b", - runner_config="1-gpu-large", -) +register_cuda_ci(est_time=122, stage="extra-a", runner_config="1-gpu-large") NUM_TURNS = 150 INPUT_LEN = 16 diff --git a/test/registered/sessions/test_streaming_session.py b/test/registered/sessions/test_streaming_session.py index 9d54a4c4b..9fce4c94b 100644 --- a/test/registered/sessions/test_streaming_session.py +++ b/test/registered/sessions/test_streaming_session.py @@ -1,1092 +1,82 @@ -import asyncio -import json -import time +"""Per-commit streaming-session tests. + +Default config + EagleV2RetractLargePage + abort-leak repro stay per-commit. +Other variants (Retract / Eagle / EagleV2 / EagleRetractLargePage) live in +test_streaming_session_extra.py. +""" + import unittest -from typing import Any, Optional -import aiohttp -import requests - -from sglang.srt.environ import envs -from sglang.srt.utils import kill_process_tree -from sglang.srt.utils.hf_transformers_utils import get_tokenizer from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.streaming_session_kit import ( + AbortLeakReproKitMixin, + StreamingSessionKitMixin, +) +from sglang.test.server_fixtures.streaming_session_fixture import ( + ABORT_REPRO_CHUNKED_PREFILL_SIZE, + ABORT_REPRO_CONTEXT_LEN, + ABORT_REPRO_PAGE_SIZE, + StreamingSessionServerBase, +) from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE3, - DEFAULT_SMALL_MODEL_NAME_FOR_TEST, DEFAULT_TARGET_MODEL_EAGLE3, - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, ) register_cuda_ci(est_time=691, stage="stage-b", runner_config="1-gpu-large") -LOGPROB_PROMPTS = [ - "The quick brown fox jumps over the lazy dog.", - "Pack my box with five dozen liquor jugs.", - "How vexingly quick daft zebras jump.", - "Sphinx of black quartz judge my vow.", - "The five boxing wizards jump quickly.", -] -# Long enough to trigger chunked prefill at 200+ tokens per slice. -LEAK_FILLER = ( - "The quick brown fox jumps over the lazy dog. " - "Pack my box with five dozen liquor jugs. " - "How vexingly quick daft zebras jump. " - "Sphinx of black quartz, judge my vow. " - "The five boxing wizards jump quickly. " - "Jackdaws love my big sphinx of quartz. " - "A wizard's job is to vex chumps quickly in fog. " - "We promptly judged antique ivory buckles for the next prize. " -) * 20 +class TestStreamingSession(StreamingSessionServerBase, StreamingSessionKitMixin): + """Default streaming-session config (small model, no spec).""" -ABORT_REPRO_CONTEXT_LEN = 512 -ABORT_REPRO_PAGE_SIZE = 256 -ABORT_REPRO_GEN_LEN = 4 -ABORT_REPRO_SESSIONS = 4 -ABORT_REPRO_WARMUP_TURNS = 1 -ABORT_REPRO_ROUNDS = 8 -ABORT_REPRO_STREAM_TOKENS = 16 -ABORT_REPRO_ABORT_TOKENS = 600 -ABORT_REPRO_NON_STREAMING_TOKENS = 16 -ABORT_REPRO_CHUNKED_PREFILL_SIZE = 4096 - -CONCURRENT_LOGPROB_SESSIONS = 6 -CONCURRENT_LOGPROB_TURNS = 5 -CONCURRENT_LOGPROB_ROUNDS = 10 - -STRESS_NUM_SESSIONS = 8 -STRESS_NUM_NON_STREAMING = 4 -STRESS_NUM_TURNS = 6 -STRESS_GEN_LEN = 16 - - -def _make_token_sized_ids( - tokenizer: Any, prefix: str, min_tokens: int, max_tokens: Optional[int] = None -) -> list[int]: - text = prefix - chunk = " pack quartz wizard sphinx zebra fox " * 16 - token_ids = tokenizer.encode(text) - while len(token_ids) < min_tokens: - text += chunk - token_ids = tokenizer.encode(text) - if max_tokens is not None: - token_ids = token_ids[:max_tokens] - return token_ids - - -async def _abort_repro_generate( - base_url: str, - session: aiohttp.ClientSession, - input_ids: list[int], - max_new_tokens: int, - session_params: Optional[dict[str, Any]] = None, - expect_abort: bool = False, -) -> Optional[dict[str, Any]]: - payload: dict[str, Any] = { - "input_ids": input_ids, - "sampling_params": { - "temperature": 0, - "max_new_tokens": max_new_tokens, - "no_stop_trim": True, - "skip_special_tokens": False, - }, - } - if session_params: - payload["session_params"] = session_params - - async with session.post(base_url + "/generate", json=payload) as resp: - text = await resp.text() - if expect_abort: - if resp.status == 200: - data = json.loads(text) - finish_reason = data.get("meta_info", {}).get("finish_reason", {}) - assert finish_reason.get("type") == "abort", text - assert "maximum allowed length" in finish_reason.get( - "message", "" - ) or "context length" in finish_reason.get("message", ""), text - return data - assert resp.status == 400, text - assert "maximum allowed length" in text or "context length" in text, text - return None - - assert resp.status == 200, text - data = json.loads(text) - finish_reason = data.get("meta_info", {}).get("finish_reason", {}) - assert finish_reason.get("type") != "abort", text - return data - - -async def _abort_repro_run_all(base_url: str, tokenizer: Any) -> None: - timeout = aiohttp.ClientTimeout(total=300) - async with aiohttp.ClientSession(timeout=timeout) as http: - session_ids = [] - for _ in range(ABORT_REPRO_SESSIONS): - async with http.post( - base_url + "/open_session", - json={"capacity_of_str_len": 50000, "streaming": True}, - ) as resp: - assert resp.status == 200, await resp.text() - session_ids.append(await resp.json()) - - try: - for warmup_turn in range(ABORT_REPRO_WARMUP_TURNS): - warmup_tasks = [] - for session_idx, session_id in enumerate(session_ids): - input_ids = _make_token_sized_ids( - tokenizer, - prefix=f"[warmup={warmup_turn} session={session_idx}]", - min_tokens=ABORT_REPRO_STREAM_TOKENS, - max_tokens=ABORT_REPRO_STREAM_TOKENS + 8, - ) - warmup_tasks.append( - _abort_repro_generate( - base_url, - http, - input_ids, - ABORT_REPRO_GEN_LEN, - session_params={"id": session_id, "rid": None}, - ) - ) - await asyncio.gather(*warmup_tasks) - - for round_idx in range(ABORT_REPRO_ROUNDS): - mixed_tasks = [] - for session_idx, session_id in enumerate(session_ids): - input_ids = _make_token_sized_ids( - tokenizer, - prefix=f"[round={round_idx} ok session={session_idx}]", - min_tokens=ABORT_REPRO_STREAM_TOKENS, - max_tokens=ABORT_REPRO_STREAM_TOKENS + 8, - ) - mixed_tasks.append( - _abort_repro_generate( - base_url, - http, - input_ids, - ABORT_REPRO_GEN_LEN, - session_params={"id": session_id, "rid": None}, - ) - ) - - for ns_idx in range(2): - input_ids = _make_token_sized_ids( - tokenizer, - prefix=f"[round={round_idx} ns={ns_idx}]", - min_tokens=ABORT_REPRO_NON_STREAMING_TOKENS, - max_tokens=ABORT_REPRO_NON_STREAMING_TOKENS + 8, - ) - mixed_tasks.append( - _abort_repro_generate( - base_url, - http, - input_ids, - ABORT_REPRO_GEN_LEN, - ) - ) - await asyncio.gather(*mixed_tasks) - - abort_tasks = [] - for session_idx, session_id in enumerate(session_ids): - input_ids = _make_token_sized_ids( - tokenizer, - prefix=f"[round={round_idx} abort session={session_idx}]", - min_tokens=ABORT_REPRO_ABORT_TOKENS, - ) - abort_tasks.append( - _abort_repro_generate( - base_url, - http, - input_ids, - ABORT_REPRO_GEN_LEN, - session_params={"id": session_id, "rid": None}, - expect_abort=True, - ) - ) - await asyncio.gather(*abort_tasks) - - recovery_tasks = [] - for session_idx, session_id in enumerate(session_ids): - input_ids = _make_token_sized_ids( - tokenizer, - prefix=f"[round={round_idx} recover session={session_idx}]", - min_tokens=ABORT_REPRO_NON_STREAMING_TOKENS, - max_tokens=ABORT_REPRO_NON_STREAMING_TOKENS + 8, - ) - recovery_tasks.append( - _abort_repro_generate( - base_url, - http, - input_ids, - ABORT_REPRO_GEN_LEN, - session_params={"id": session_id, "rid": None}, - ) - ) - recovery_results = await asyncio.gather(*recovery_tasks) - for result in recovery_results: - assert result is not None - assert result["meta_info"]["cached_tokens"] > 0, result - - health = requests.get(base_url + "/health", timeout=10) - if health.status_code != 200: - raise RuntimeError( - f"server unhealthy after round={round_idx}: " - f"{health.status_code} {health.text}" - ) - finally: - for session_id in session_ids: - async with http.post( - base_url + "/close_session", json={"session_id": session_id} - ) as resp: - assert resp.status == 200, await resp.text() - - -async def _async_generate( - base_url: str, - session: aiohttp.ClientSession, - input_ids: list[int], - max_new_tokens: int = 8, - session_params: Optional[dict[str, Any]] = None, - return_logprob: bool = False, - logprob_start_len: Optional[int] = None, -) -> dict[str, Any]: - payload: dict[str, Any] = { - "input_ids": input_ids, - "sampling_params": { - "temperature": 0, - "max_new_tokens": max_new_tokens, - "no_stop_trim": True, - "skip_special_tokens": False, - }, - } - if session_params: - payload["session_params"] = session_params - if return_logprob: - payload["return_logprob"] = True - if logprob_start_len is not None: - payload["logprob_start_len"] = logprob_start_len - timeout = aiohttp.ClientTimeout(total=300) - async with session.post( - base_url + "/generate", json=payload, timeout=timeout - ) as resp: - assert resp.status == 200, f"Generate failed: {await resp.text()}" - return await resp.json() - - -async def _concurrent_logprob_run(base_url: str, tokenizer: Any, **gen_kwargs) -> None: - """N sessions per round, all requests fired simultaneously per turn so - the running batch has real concurrency (retract can actually kick one). - """ - timeout = aiohttp.ClientTimeout(total=300) - async with aiohttp.ClientSession(timeout=timeout) as http: - for _ in range(CONCURRENT_LOGPROB_ROUNDS): - sids: list[str] = [] - for _ in range(CONCURRENT_LOGPROB_SESSIONS): - async with http.post( - base_url + "/open_session", - json={"capacity_of_str_len": 50000, "streaming": True}, - ) as resp: - assert resp.status == 200 - sids.append(await resp.json()) - - rids: list[Optional[str]] = [None] * CONCURRENT_LOGPROB_SESSIONS - for turn in range(CONCURRENT_LOGPROB_TURNS): - tasks = [] - for s in range(CONCURRENT_LOGPROB_SESSIONS): - text = ( - f"S{s} T{turn}: " - f"{LOGPROB_PROMPTS[turn % len(LOGPROB_PROMPTS)]}" - ) - ids = tokenizer.encode(text) - tasks.append( - _async_generate( - base_url, - http, - ids, - session_params={"id": sids[s], "rid": rids[s]}, - **gen_kwargs, - ) - ) - results = await asyncio.gather(*tasks) - for s in range(CONCURRENT_LOGPROB_SESSIONS): - rids[s] = results[s]["meta_info"]["id"] - - for sid in sids: - async with http.post( - base_url + "/close_session", json={"session_id": sid} - ) as resp: - assert resp.status == 200 - - -async def _stress_run_all(base_url: str, tokenizer: Any) -> None: - """Streaming + non-streaming mixed batches under retract pressure. - Long prompts (~200+ tokens) trigger chunked prefill so retract can - interrupt mid-extend. - """ - timeout = aiohttp.ClientTimeout(total=300) - async with aiohttp.ClientSession(timeout=timeout) as http: - sids: list[str] = [] - for _ in range(STRESS_NUM_SESSIONS): - async with http.post( - base_url + "/open_session", - json={"capacity_of_str_len": 50000, "streaming": True}, - ) as resp: - assert resp.status == 200 - sids.append(await resp.json()) - - rids: list[Optional[str]] = [None] * STRESS_NUM_SESSIONS - for turn in range(STRESS_NUM_TURNS): - tasks = [] - # Streaming requests — long prompts to trigger chunked prefill. - for s in range(STRESS_NUM_SESSIONS): - offset = (s * STRESS_NUM_TURNS + turn) * 200 - text = ( - f"Session {s} turn {turn}: " f"{LEAK_FILLER[offset : offset + 800]}" - ) - ids = tokenizer.encode(text) - tasks.append( - _async_generate( - base_url, - http, - ids, - max_new_tokens=STRESS_GEN_LEN, - session_params={"id": sids[s], "rid": rids[s]}, - ) - ) - - # Non-streaming requests interleaved. - for ns in range(STRESS_NUM_NON_STREAMING): - text = ( - f"Non-streaming {ns} turn {turn}: " - f"{LEAK_FILLER[ns * 100 : ns * 100 + 400]}" - ) - ids = tokenizer.encode(text) - tasks.append( - _async_generate( - base_url, - http, - ids, - max_new_tokens=STRESS_GEN_LEN, - ) - ) - - results = await asyncio.gather(*tasks) - for s in range(STRESS_NUM_SESSIONS): - rids[s] = results[s]["meta_info"]["id"] - - for sid in sids: - async with http.post( - base_url + "/close_session", json={"session_id": sid} - ) as resp: - assert resp.status == 200 - - -class TestStreamingSession(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST - cls.base_url = DEFAULT_URL_FOR_TEST - with envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(2): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enable-streaming-session", - "--chunked-prefill-size", - "512", - ], - ) - cls.tokenizer = get_tokenizer(cls.model) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - # -1 for non-overlap subclasses: the last sampled token isn't committed - # before max_new stops, so slot.kv_committed_len = input + output - 1. - kv_inherit_offset = 0 - - def test_kv_cache_inheritance(self, gen_len=12): - """Each turn's cached_tokens must equal previous turn's prompt+completion - (modulo kv_inherit_offset).""" - chunks = [ - "Let me tell you something about France.", - "The capital of France is", - "The population of the city is", - ] - chunks_ids = [self.tokenizer.encode(x) for x in chunks] - for i in range(1, len(chunks_ids)): - if chunks_ids[i][0] == self.tokenizer.bos_token_id: - chunks_ids[i] = chunks_ids[i][1:] - - # === Part 1: streaming session — check KV inheritance === - requests.post(self.base_url + "/flush_cache") - session_id = requests.post( - self.base_url + "/open_session", - json={"capacity_of_str_len": 1000, "streaming": True}, - ).json() - rid = None - - prev_kv_len = 0 - for turn_idx, chunk_ids in enumerate(chunks_ids): - response = requests.post( - self.base_url + "/generate", - json={ - "input_ids": chunk_ids, - "session_params": {"id": session_id, "rid": rid}, - "sampling_params": { - "temperature": 0, - "max_new_tokens": gen_len, - "no_stop_trim": True, - "skip_special_tokens": False, - }, - }, - ).json() - rid = response["meta_info"]["id"] - cached = response["meta_info"]["cached_tokens"] - prompt_tokens = response["meta_info"]["prompt_tokens"] - completion_tokens = response["meta_info"]["completion_tokens"] - - if turn_idx == 0: - # Turn 1: cache flushed, no hit. - self.assertEqual(cached, 0, "Turn 1: clean start, no cache hit") - else: - # Turns 2+: cached_tokens reflects KV inherited from previous turn - # (via inherit_kv_states, not radix tree matching). - expected = prev_kv_len + self.kv_inherit_offset - self.assertEqual( - cached, - expected, - f"Turn {turn_idx + 1}: inherited {cached} != expected {expected}", - ) - prev_kv_len = prompt_tokens + completion_tokens - - # Close the session. - ret = requests.post( - self.base_url + "/close_session", - json={"session_id": session_id}, - ) - self.assertEqual(ret.status_code, 200) - - def test_leak_logprob_concurrent(self) -> None: - """Concurrent multi-session × 3 logprob modes (output / input / none), - watch for KV leak.""" - requests.post(self.base_url + "/flush_cache") - # Output logprob - asyncio.run( - _concurrent_logprob_run(self.base_url, self.tokenizer, return_logprob=True) - ) - # Input logprob (logprob_start_len=0) - asyncio.run( - _concurrent_logprob_run( - self.base_url, - self.tokenizer, - return_logprob=True, - logprob_start_len=0, - ) - ) - # No logprob - asyncio.run(_concurrent_logprob_run(self.base_url, self.tokenizer)) - time.sleep(3) - assert ( - requests.get(self.base_url + "/health").status_code == 200 - ), "Server unhealthy after concurrent logprob sessions." - - def test_stress_concurrent_sessions(self) -> None: - """High concurrency streaming + non-streaming with retract pressure; - scheduler must roll back streaming KV without leaking.""" - requests.post(self.base_url + "/flush_cache") - asyncio.run(_stress_run_all(self.base_url, self.tokenizer)) - - for i in range(3): - ids = self.tokenizer.encode(f"Post-stress cleanup {i}.") - requests.post( - self.base_url + "/generate", - json={ - "input_ids": ids, - "sampling_params": {"temperature": 0, "max_new_tokens": 4}, - }, - ) - - time.sleep(5) - health = requests.get(self.base_url + "/health") - self.assertEqual( - health.status_code, - 200, - "Server unhealthy after concurrent stress test — " - "likely a token leak from retract/mixed-chunk + streaming session.", - ) - - def test_nth_mid_abort_recovery(self) -> None: - """Abort an Nth-turn request mid-decode; session rolls back to last - successful turn.""" - requests.post(self.base_url + "/flush_cache") - - resp = requests.post( - self.base_url + "/open_session", - json={"capacity_of_str_len": 50000, "streaming": True}, - ) - self.assertEqual(resp.status_code, 200) - session_id = resp.json() - - try: - # Turn 1: normal generate to create slot. - ids_1 = self.tokenizer.encode("Tell me a very long story about a wizard.") - resp_1 = requests.post( - self.base_url + "/generate", - json={ - "input_ids": ids_1, - "sampling_params": {"temperature": 0, "max_new_tokens": 16}, - "session_params": {"id": session_id, "rid": None}, - }, - timeout=30, - ) - self.assertEqual(resp_1.status_code, 200, resp_1.text) - data_1 = resp_1.json() - turn_1_total = ( - data_1["meta_info"]["prompt_tokens"] - + data_1["meta_info"]["completion_tokens"] - ) - - # Turn 2: long generate, then abort mid-decode. - ids_2 = self.tokenizer.encode(" Continue the story in great detail.") - - import threading - - result = [None] - - def do_generate(): - r = requests.post( - self.base_url + "/generate", - json={ - "input_ids": ids_2, - "sampling_params": { - "temperature": 0, - "max_new_tokens": 100000, - }, - "session_params": {"id": session_id, "rid": None}, - }, - timeout=60, - ) - result[0] = r - - t = threading.Thread(target=do_generate) - t.start() - time.sleep(0.5) - abort_resp = requests.post( - self.base_url + "/abort_request", - json={"rid": "", "abort_all": True}, - timeout=10, - ) - self.assertEqual(abort_resp.status_code, 200, abort_resp.text) - t.join(timeout=30) - - self.assertIsNotNone(result[0], "Turn 2 should have returned") - data_2 = result[0].json() - self.assertEqual( - data_2["meta_info"]["finish_reason"]["type"], - "abort", - "Turn 2 should be aborted, not finished normally", - ) - - # Turn 3: recovery. Rolls back to turn 1. - ids_3 = self.tokenizer.encode(" What happens next?") - for attempt in range(20): - resp_3 = requests.post( - self.base_url + "/generate", - json={ - "input_ids": ids_3, - "sampling_params": {"temperature": 0, "max_new_tokens": 8}, - "session_params": {"id": session_id, "rid": None}, - }, - timeout=30, - ) - if resp_3.status_code == 200: - break - time.sleep(0.5) - self.assertEqual(resp_3.status_code, 200, resp_3.text) - data_3 = resp_3.json() - # prompt_tokens = turn_1_total + append (BOS stripped). - bos = 1 if ids_3[0] == self.tokenizer.bos_token_id else 0 - expected_prompt_3 = turn_1_total + len(ids_3) - bos - self.assertEqual( - data_3["meta_info"]["prompt_tokens"], - expected_prompt_3, - "prompt_tokens must equal turn_1_total + append (no stale abort context)", - ) - finally: - requests.post( - self.base_url + "/close_session", - json={"session_id": session_id}, - ) - - health = requests.get(self.base_url + "/health", timeout=10) - self.assertEqual(health.status_code, 200) - - def test_first_mid_abort_recovery(self) -> None: - """Abort the very first request mid-decode (no slot yet; ephemeral - slot is created and nuked). Session must still be usable.""" - requests.post(self.base_url + "/flush_cache") - - resp = requests.post( - self.base_url + "/open_session", - json={"capacity_of_str_len": 50000, "streaming": True}, - ) - self.assertEqual(resp.status_code, 200) - session_id = resp.json() - - try: - ids_1 = self.tokenizer.encode("Tell me a very long story about a wizard.") - - import threading - - result = [None] - - def do_generate(): - r = requests.post( - self.base_url + "/generate", - json={ - "input_ids": ids_1, - "sampling_params": { - "temperature": 0, - "max_new_tokens": 100000, - }, - "session_params": {"id": session_id, "rid": None}, - }, - timeout=60, - ) - result[0] = r - - t = threading.Thread(target=do_generate) - t.start() - time.sleep(0.5) - abort_resp = requests.post( - self.base_url + "/abort_request", - json={"rid": "", "abort_all": True}, - timeout=10, - ) - self.assertEqual(abort_resp.status_code, 200, abort_resp.text) - t.join(timeout=30) - - self.assertIsNotNone(result[0], "Turn 1 should have returned") - data_1 = result[0].json() - self.assertEqual( - data_1["meta_info"]["finish_reason"]["type"], - "abort", - "Turn 1 should be aborted, not finished normally", - ) - - # Turn 2: recovery. No inherited context (req_nodes empty). - ids_2 = self.tokenizer.encode("Tell me a short joke.") - for attempt in range(20): - resp_2 = requests.post( - self.base_url + "/generate", - json={ - "input_ids": ids_2, - "sampling_params": {"temperature": 0, "max_new_tokens": 8}, - "session_params": {"id": session_id, "rid": None}, - }, - timeout=30, - ) - if resp_2.status_code == 200: - break - time.sleep(0.5) - self.assertEqual(resp_2.status_code, 200, resp_2.text) - data_2 = resp_2.json() - self.assertEqual( - data_2["meta_info"]["prompt_tokens"], - len(ids_2), - "prompt_tokens must equal turn 2 input only (no inherited context)", - ) - finally: - requests.post( - self.base_url + "/close_session", - json={"session_id": session_id}, - ) - - health = requests.get(self.base_url + "/health", timeout=10) - self.assertEqual(health.status_code, 200) - - def test_preabort_recovery(self) -> None: - """Pre-abort (rejected by create_req) preserves the slot; next turn - inherits correctly.""" - requests.post(self.base_url + "/flush_cache") - - resp = requests.post( - self.base_url + "/open_session", - json={"capacity_of_str_len": 50000, "streaming": True}, - ) - self.assertEqual(resp.status_code, 200) - session_id = resp.json() - - try: - # Turn 1: normal generate to create slot. - ids_1 = self.tokenizer.encode("Tell me a very long story about a wizard.") - resp_1 = requests.post( - self.base_url + "/generate", - json={ - "input_ids": ids_1, - "sampling_params": {"temperature": 0, "max_new_tokens": 16}, - "session_params": {"id": session_id, "rid": None}, - }, - timeout=30, - ) - self.assertEqual(resp_1.status_code, 200, resp_1.text) - data_1 = resp_1.json() - turn_1_total = ( - data_1["meta_info"]["prompt_tokens"] - + data_1["meta_info"]["completion_tokens"] - ) - - # Turn 2: pre-aborted via unsupported offset parameter. - ids_2 = self.tokenizer.encode(" This should be rejected.") - resp_2 = requests.post( - self.base_url + "/generate", - json={ - "input_ids": ids_2, - "sampling_params": {"temperature": 0, "max_new_tokens": 8}, - "session_params": { - "id": session_id, - "rid": None, - "offset": 1, - }, - }, - timeout=30, - ) - self.assertIn(resp_2.status_code, (200, 400), resp_2.text) - - # Turn 3: normal append. Slot should be intact from turn 1. - ids_3 = self.tokenizer.encode(" What happens next?") - resp_3 = requests.post( - self.base_url + "/generate", - json={ - "input_ids": ids_3, - "sampling_params": {"temperature": 0, "max_new_tokens": 8}, - "session_params": {"id": session_id, "rid": None}, - }, - timeout=30, - ) - self.assertEqual(resp_3.status_code, 200, resp_3.text) - data_3 = resp_3.json() - bos = 1 if ids_3[0] == self.tokenizer.bos_token_id else 0 - expected_prompt_3 = turn_1_total + len(ids_3) - bos - self.assertEqual( - data_3["meta_info"]["prompt_tokens"], - expected_prompt_3, - "prompt_tokens must equal turn_1_total + append (slot preserved)", - ) - finally: - requests.post( - self.base_url + "/close_session", - json={"session_id": session_id}, - ) - - health = requests.get(self.base_url + "/health", timeout=10) - self.assertEqual(health.status_code, 200) - - -class TestStreamingSessionRetractMixedChunk(TestStreamingSession): - """Retract + --enable-mixed-chunk.""" - - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST - cls.base_url = DEFAULT_URL_FOR_TEST - with ( - envs.SGLANG_TEST_RETRACT.override(True), - envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(2), - ): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enable-streaming-session", - "--chunked-prefill-size", - "128", - "--enable-mixed-chunk", - ], - ) - cls.tokenizer = get_tokenizer(cls.model) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - -class TestStreamingSessionRetractLargePage(TestStreamingSession): - """Retract + page=256: exercises page-aligned `_free_tail`. Partial-page - free would corrupt pages still holding committed tokens.""" - - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST - cls.base_url = DEFAULT_URL_FOR_TEST - with ( - envs.SGLANG_TEST_RETRACT.override(True), - envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(2), - ): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enable-streaming-session", - "--chunked-prefill-size", - "4096", - "--page-size", - "256", - ], - ) - cls.tokenizer = get_tokenizer(cls.model) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - -class TestStreamingSessionEagle(TestStreamingSession): - """EAGLE3 spec v1 (overlap disabled); offset=-1 — see base class note.""" - - kv_inherit_offset = -1 - - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_TARGET_MODEL_EAGLE3 - cls.base_url = DEFAULT_URL_FOR_TEST - with ( - envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(2), - envs.SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN.override(True), - ): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enable-streaming-session", - "--disable-overlap-schedule", - "--chunked-prefill-size", - "512", - "--dtype=float16", - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model", - DEFAULT_DRAFT_MODEL_EAGLE3, - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--mem-fraction-static", - "0.7", - ], - ) - cls.tokenizer = get_tokenizer(cls.model) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - -class TestStreamingSessionEagleV2(TestStreamingSession): - """EAGLE3 spec v2 (overlap on).""" - - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_TARGET_MODEL_EAGLE3 - cls.base_url = DEFAULT_URL_FOR_TEST - with ( - envs.SGLANG_ENABLE_SPEC_V2.override(True), - envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(2), - envs.SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN.override(True), - ): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enable-streaming-session", - "--chunked-prefill-size", - "512", - "--dtype=float16", - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model", - DEFAULT_DRAFT_MODEL_EAGLE3, - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--mem-fraction-static", - "0.7", - ], - ) - cls.tokenizer = get_tokenizer(cls.model) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - -class TestStreamingSessionEagleRetractLargePage(TestStreamingSession): - """EAGLE3 spec v1 + retract + page=256: max-pressure on `_free_tail` - (spec tail + retract alloc-commit gap + page alignment).""" - - kv_inherit_offset = -1 - - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_TARGET_MODEL_EAGLE3 - cls.base_url = DEFAULT_URL_FOR_TEST - with ( - envs.SGLANG_TEST_RETRACT.override(True), - envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(2), - envs.SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN.override(True), - ): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enable-streaming-session", - "--disable-overlap-schedule", - "--chunked-prefill-size", - "4096", - "--dtype=float16", - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model", - DEFAULT_DRAFT_MODEL_EAGLE3, - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--mem-fraction-static", - "0.7", - "--page-size", - "256", - ], - ) - cls.tokenizer = get_tokenizer(cls.model) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) + extra_args = ["--chunked-prefill-size", "512"] class TestStreamingSessionEagleV2RetractLargePage(TestStreamingSession): """EAGLE3 spec v2 + retract + page=256.""" - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_TARGET_MODEL_EAGLE3 - cls.base_url = DEFAULT_URL_FOR_TEST - with ( - envs.SGLANG_ENABLE_SPEC_V2.override(True), - envs.SGLANG_TEST_RETRACT.override(True), - envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(2), - envs.SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN.override(True), - ): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enable-streaming-session", - "--chunked-prefill-size", - "4096", - "--dtype=float16", - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model", - DEFAULT_DRAFT_MODEL_EAGLE3, - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--mem-fraction-static", - "0.7", - "--page-size", - "256", - ], - ) - cls.tokenizer = get_tokenizer(cls.model) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) + model = DEFAULT_TARGET_MODEL_EAGLE3 + extra_args = [ + "--chunked-prefill-size", + "4096", + "--dtype=float16", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model", + DEFAULT_DRAFT_MODEL_EAGLE3, + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-fraction-static", + "0.7", + "--page-size", + "256", + ] + env_overrides = [ + ("SGLANG_ENABLE_SPEC_V2", True), + ("SGLANG_TEST_RETRACT", True), + ("SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN", True), + ] -class TestStreamingSessionAbortLeakRepro(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST - cls.base_url = DEFAULT_URL_FOR_TEST - with envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(2): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enable-streaming-session", - "--chunked-prefill-size", - str(ABORT_REPRO_CHUNKED_PREFILL_SIZE), - "--context-length", - str(ABORT_REPRO_CONTEXT_LEN), - "--page-size", - str(ABORT_REPRO_PAGE_SIZE), - "--max-running-requests", - "32", - "--log-level", - "info", - ], - ) - cls.tokenizer = get_tokenizer(cls.model) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_abort_heavy_chunked_prefill_does_not_leak(self) -> None: - requests.post(self.base_url + "/flush_cache") - - asyncio.run(_abort_repro_run_all(self.base_url, self.tokenizer)) - - for i in range(3): - ids = self.tokenizer.encode(f"Post-session cleanup request {i}.") - response = requests.post( - self.base_url + "/generate", - json={ - "input_ids": ids, - "sampling_params": {"temperature": 0, "max_new_tokens": 4}, - }, - timeout=30, - ) - self.assertEqual(response.status_code, 200, response.text) - - time.sleep(5) - self.assertIsNone( - self.process.poll(), - "Server crashed during abort-heavy streaming session repro.", - ) - - health = requests.get(self.base_url + "/health", timeout=10) - self.assertEqual( - health.status_code, - 200, - "Server unhealthy after abort-heavy streaming session cleanup.", - ) +class TestStreamingSessionAbortLeakRepro( + StreamingSessionServerBase, AbortLeakReproKitMixin +): + extra_args = [ + "--chunked-prefill-size", + str(ABORT_REPRO_CHUNKED_PREFILL_SIZE), + "--context-length", + str(ABORT_REPRO_CONTEXT_LEN), + "--page-size", + str(ABORT_REPRO_PAGE_SIZE), + "--max-running-requests", + "32", + "--log-level", + "info", + ] if __name__ == "__main__": diff --git a/test/registered/sessions/test_streaming_session_extra.py b/test/registered/sessions/test_streaming_session_extra.py new file mode 100644 index 000000000..582bc4740 --- /dev/null +++ b/test/registered/sessions/test_streaming_session_extra.py @@ -0,0 +1,105 @@ +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.streaming_session_kit import StreamingSessionKitMixin +from sglang.test.server_fixtures.streaming_session_fixture import ( + StreamingSessionServerBase, +) +from sglang.test.test_utils import ( + DEFAULT_DRAFT_MODEL_EAGLE3, + DEFAULT_TARGET_MODEL_EAGLE3, +) + +register_cuda_ci(est_time=691, stage="extra-a", runner_config="1-gpu-large") + + +class TestStreamingSessionRetractMixedChunk( + StreamingSessionServerBase, StreamingSessionKitMixin +): + """Retract + --enable-mixed-chunk.""" + + extra_args = ["--chunked-prefill-size", "128", "--enable-mixed-chunk"] + env_overrides = [("SGLANG_TEST_RETRACT", True)] + + +class TestStreamingSessionRetractLargePage( + StreamingSessionServerBase, StreamingSessionKitMixin +): + """Retract + page=256: exercises page-aligned `_free_tail`. Partial-page + free would corrupt pages still holding committed tokens.""" + + extra_args = ["--chunked-prefill-size", "4096", "--page-size", "256"] + env_overrides = [("SGLANG_TEST_RETRACT", True)] + + +# Common EAGLE3 spec args; reused by Eagle/EagleV2/EagleRetractLargePage variants. +_EAGLE3_SPEC_ARGS = [ + "--dtype=float16", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model", + DEFAULT_DRAFT_MODEL_EAGLE3, + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-fraction-static", + "0.7", +] + + +class TestStreamingSessionEagle(StreamingSessionServerBase, StreamingSessionKitMixin): + """EAGLE3 spec v1 (overlap disabled); offset=-1 — see kit's note.""" + + kv_inherit_offset = -1 + model = DEFAULT_TARGET_MODEL_EAGLE3 + extra_args = [ + "--disable-overlap-schedule", + "--chunked-prefill-size", + "512", + *_EAGLE3_SPEC_ARGS, + ] + env_overrides = [("SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN", True)] + + +class TestStreamingSessionEagleV2(StreamingSessionServerBase, StreamingSessionKitMixin): + """EAGLE3 spec v2 (overlap on).""" + + model = DEFAULT_TARGET_MODEL_EAGLE3 + extra_args = [ + "--chunked-prefill-size", + "512", + *_EAGLE3_SPEC_ARGS, + ] + env_overrides = [ + ("SGLANG_ENABLE_SPEC_V2", True), + ("SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN", True), + ] + + +class TestStreamingSessionEagleRetractLargePage( + StreamingSessionServerBase, StreamingSessionKitMixin +): + """EAGLE3 spec v1 + retract + page=256: max-pressure on `_free_tail` + (spec tail + retract alloc-commit gap + page alignment).""" + + kv_inherit_offset = -1 + model = DEFAULT_TARGET_MODEL_EAGLE3 + extra_args = [ + "--disable-overlap-schedule", + "--chunked-prefill-size", + "4096", + *_EAGLE3_SPEC_ARGS, + "--page-size", + "256", + ] + env_overrides = [ + ("SGLANG_TEST_RETRACT", True), + ("SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN", True), + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/sessions/test_streaming_session_swa.py b/test/registered/sessions/test_streaming_session_swa.py index 89c75e955..b9b17d4d6 100644 --- a/test/registered/sessions/test_streaming_session_swa.py +++ b/test/registered/sessions/test_streaming_session_swa.py @@ -1,27 +1,15 @@ -import os -import sys import unittest -from sglang.srt.environ import envs -from sglang.srt.utils import kill_process_tree -from sglang.srt.utils.hf_transformers_utils import get_tokenizer from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - popen_launch_server, +from sglang.test.kits.streaming_session_kit import ( + AbortLeakReproKitMixin, + StreamingSessionKitMixin, ) - -# test/ has no __init__.py; add sibling dir so sibling module is importable -# when this file is run as a script via `python3 `. -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -from test_streaming_session import ( # noqa: E402 +from sglang.test.server_fixtures.streaming_session_fixture import ( ABORT_REPRO_CHUNKED_PREFILL_SIZE, ABORT_REPRO_CONTEXT_LEN, ABORT_REPRO_PAGE_SIZE, - TestStreamingSession, - TestStreamingSessionAbortLeakRepro, + StreamingSessionServerBase, ) register_cuda_ci(est_time=519, stage="stage-b", runner_config="1-gpu-large") @@ -37,125 +25,63 @@ SWA_COMMON_ARGS = [ ] -class TestStreamingSessionSWA(TestStreamingSession): +class TestStreamingSessionSWA(StreamingSessionServerBase, StreamingSessionKitMixin): """Baseline streaming session on a hybrid-SWA model.""" - @classmethod - def setUpClass(cls): - cls.model = SWA_MODEL - cls.base_url = DEFAULT_URL_FOR_TEST - with envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(2): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enable-streaming-session", - "--chunked-prefill-size", - "512", - *SWA_COMMON_ARGS, - ], - ) - cls.tokenizer = get_tokenizer(cls.model) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) + model = SWA_MODEL + extra_args = ["--chunked-prefill-size", "512", *SWA_COMMON_ARGS] -class TestStreamingSessionSWARetractLargePage(TestStreamingSession): +class TestStreamingSessionSWARetractLargePage( + StreamingSessionServerBase, StreamingSessionKitMixin +): """SWA under retract decode with page=256.""" - @classmethod - def setUpClass(cls): - cls.model = SWA_MODEL - cls.base_url = DEFAULT_URL_FOR_TEST - with ( - envs.SGLANG_TEST_RETRACT.override(True), - envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(2), - ): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enable-streaming-session", - "--chunked-prefill-size", - "4096", - "--page-size", - "256", - *SWA_COMMON_ARGS, - ], - ) - cls.tokenizer = get_tokenizer(cls.model) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) + model = SWA_MODEL + extra_args = [ + "--chunked-prefill-size", + "4096", + "--page-size", + "256", + *SWA_COMMON_ARGS, + ] + env_overrides = [("SGLANG_TEST_RETRACT", True)] -class TestStreamingSessionSWARetractMixedChunk(TestStreamingSession): +class TestStreamingSessionSWARetractMixedChunk( + StreamingSessionServerBase, StreamingSessionKitMixin +): """SWA under retract decode with --enable-mixed-chunk.""" - @classmethod - def setUpClass(cls): - cls.model = SWA_MODEL - cls.base_url = DEFAULT_URL_FOR_TEST - with ( - envs.SGLANG_TEST_RETRACT.override(True), - envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(2), - ): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enable-streaming-session", - "--chunked-prefill-size", - "128", - "--enable-mixed-chunk", - *SWA_COMMON_ARGS, - ], - ) - cls.tokenizer = get_tokenizer(cls.model) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) + model = SWA_MODEL + extra_args = [ + "--chunked-prefill-size", + "128", + "--enable-mixed-chunk", + *SWA_COMMON_ARGS, + ] + env_overrides = [("SGLANG_TEST_RETRACT", True)] -class TestStreamingSessionSWAAbortLeakRepro(TestStreamingSessionAbortLeakRepro): +class TestStreamingSessionSWAAbortLeakRepro( + StreamingSessionServerBase, AbortLeakReproKitMixin +): """SWA abort-heavy chunked prefill leak repro.""" - @classmethod - def setUpClass(cls): - cls.model = SWA_MODEL - cls.base_url = DEFAULT_URL_FOR_TEST - with envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(2): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enable-streaming-session", - "--chunked-prefill-size", - str(ABORT_REPRO_CHUNKED_PREFILL_SIZE), - "--context-length", - str(ABORT_REPRO_CONTEXT_LEN), - "--page-size", - str(ABORT_REPRO_PAGE_SIZE), - "--max-running-requests", - "32", - "--log-level", - "info", - *SWA_COMMON_ARGS, - ], - ) - cls.tokenizer = get_tokenizer(cls.model) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) + model = SWA_MODEL + extra_args = [ + "--chunked-prefill-size", + str(ABORT_REPRO_CHUNKED_PREFILL_SIZE), + "--context-length", + str(ABORT_REPRO_CONTEXT_LEN), + "--page-size", + str(ABORT_REPRO_PAGE_SIZE), + "--max-running-requests", + "32", + "--log-level", + "info", + *SWA_COMMON_ARGS, + ] if __name__ == "__main__": diff --git a/test/registered/spec/eagle/test_eagle_dp_attention.py b/test/registered/spec/eagle/test_eagle_dp_attention.py index e45457b37..b31b38ebf 100644 --- a/test/registered/spec/eagle/test_eagle_dp_attention.py +++ b/test/registered/spec/eagle/test_eagle_dp_attention.py @@ -20,8 +20,11 @@ from sglang.test.test_utils import ( write_github_step_summary, ) -# EAGLE3 with DP attention (tp=2, dp=2, requires 4 GPUs) -register_cuda_ci(est_time=99, stage="stage-c", runner_config="4-gpu-h100") +# EAGLE3 with DP attention (tp=2, dp=2, requires 4 GPUs). +# Per-commit EAGLE + DP-attn coverage on CUDA is provided by +# test_eagle_infer_beta_dp_attention.py (B200 4-gpu), so this H100 variant +# is gated to extra-b only. +register_cuda_ci(est_time=99, stage="extra-b", runner_config="4-gpu-h100") register_amd_ci(est_time=200, suite="stage-c-test-4-gpu-amd") diff --git a/test/registered/spec/eagle/test_eagle_infer_a.py b/test/registered/spec/eagle/test_eagle_infer_a.py index d845990d5..0354b958c 100644 --- a/test/registered/spec/eagle/test_eagle_infer_a.py +++ b/test/registered/spec/eagle/test_eagle_infer_a.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( CustomTestCase, ) -register_cuda_ci(est_time=357, stage="stage-b", runner_config="1-gpu-large") +register_cuda_ci(est_time=357, stage="extra-a", runner_config="1-gpu-large") class TestEAGLEEngine(CustomTestCase): diff --git a/test/registered/spec/test_spec_ngram.py b/test/registered/spec/test_spec_ngram.py new file mode 100644 index 000000000..65e001c55 --- /dev/null +++ b/test/registered/spec/test_spec_ngram.py @@ -0,0 +1,19 @@ +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.server_fixtures.ngram_fixture import NgramServerBase + +# Per-commit: Paged backend only. +# - FA3 base test archived to test/manual/spec/test_spec_ngram_fa3.py +# - Triton + Flashinfer moved to test_spec_ngram_extra.py +register_cuda_ci(est_time=254, stage="stage-b", runner_config="1-gpu-large") + + +class TestNgramSpeculativeDecodingPaged(NgramServerBase, GSM8KMixin): + attention_backend = "flashinfer" + extra_args = ["--page-size", "64"] + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/spec/test_ngram_speculative_decoding.py b/test/registered/spec/test_spec_ngram_extra.py similarity index 53% rename from test/registered/spec/test_ngram_speculative_decoding.py rename to test/registered/spec/test_spec_ngram_extra.py index b3751fd29..8a8039a13 100644 --- a/test/registered/spec/test_ngram_speculative_decoding.py +++ b/test/registered/spec/test_spec_ngram_extra.py @@ -2,83 +2,22 @@ import unittest import requests -from sglang.srt.environ import envs -from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.eval_accuracy_kit import GSM8KMixin -from sglang.test.test_utils import ( - DEFAULT_TARGET_MODEL_NGRAM, - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) +from sglang.test.server_fixtures.ngram_fixture import NgramServerBase -register_cuda_ci(est_time=254, stage="stage-b", runner_config="1-gpu-large") - -GSM_DATASET_PATH = None +# Extra: Triton + Flashinfer NGRAM backends. Sibling per-commit file +# (test_spec_ngram.py) keeps the Paged variant. +register_cuda_ci(est_time=254, stage="extra-a", runner_config="1-gpu-large") -# Default server arguments shared across all tests -DEFAULT_SERVER_ARGS = [ - "--trust-remote-code", - "--cuda-graph-max-bs", - "8", - "--speculative-algorithm", - "NGRAM", - "--speculative-num-draft-tokens", - "16", - "--mem-fraction-static", - 0.8, -] +class TestNgramSpeculativeDecodingTriton(NgramServerBase, GSM8KMixin): + attention_backend = "triton" -class TestNgramSpeculativeDecodingBase(GSM8KMixin, CustomTestCase): - model = DEFAULT_TARGET_MODEL_NGRAM - base_url = DEFAULT_URL_FOR_TEST - gsm8k_accuracy_thres = 0.79 # derived tests need to override this - gsm8k_accept_length_thres = 1.8 # derived spec decoding tests need to override this - - @classmethod - def get_server_args(cls): - """Return the arguments for the server launch. Override in subclasses.""" - return DEFAULT_SERVER_ARGS + ["--attention-backend", "fa3"] - - @classmethod - def setUpClass(cls): - # disable deep gemm precompile to make launch server faster - # please don't do this if you want to make your inference workload faster - envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False) - envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False) - model = cls.model - cls.process = popen_launch_server( - model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=cls.get_server_args(), - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - -class TestNgramSpeculativeDecodingTriton(TestNgramSpeculativeDecodingBase): - - @classmethod - def get_server_args(cls): - return DEFAULT_SERVER_ARGS + ["--attention-backend", "triton"] - - -class TestNgramSpeculativeDecodingFlashinfer(TestNgramSpeculativeDecodingBase): - @classmethod - def get_server_args(cls): - return DEFAULT_SERVER_ARGS + [ - "--attention-backend", - "flashinfer", - "--speculative-ngram-external-sam-budget", - "8", - ] +class TestNgramSpeculativeDecodingFlashinfer(NgramServerBase, GSM8KMixin): + attention_backend = "flashinfer" + extra_args = ["--speculative-ngram-external-sam-budget", "8"] def test_output_as_corpus_boosts_accept_length(self): """Baseline → HTTP add corpus → verify accept length boost.""" @@ -147,17 +86,5 @@ class TestNgramSpeculativeDecodingFlashinfer(TestNgramSpeculativeDecodingBase): ) -class TestNgramSpeculativeDecodingPaged(TestNgramSpeculativeDecodingBase): - - @classmethod - def get_server_args(cls): - return DEFAULT_SERVER_ARGS + [ - "--attention-backend", - "flashinfer", - "--page-size", - "64", - ] - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/spec/test_spec_standalone.py b/test/registered/spec/test_spec_standalone.py new file mode 100644 index 000000000..afd45017a --- /dev/null +++ b/test/registered/spec/test_spec_standalone.py @@ -0,0 +1,27 @@ +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.server_fixtures.standalone_fixture import StandaloneServerBase +from sglang.test.test_utils import CustomTestCase + +# V2 standalone speculative decoding tests (FA3, Triton, FlashInfer backends). +# Non-V2 backends moved to test_spec_standalone_extra.py. +register_cuda_ci(est_time=406, stage="stage-b", runner_config="1-gpu-large") + + +class TestStandaloneV2SpeculativeDecodingBase(StandaloneServerBase, CustomTestCase): + attention_backend = "fa3" + + +class TestStandaloneV2SpeculativeDecodingTriton(StandaloneServerBase, CustomTestCase): + attention_backend = "triton" + + +class TestStandaloneV2SpeculativeDecodingFlashinfer( + StandaloneServerBase, CustomTestCase +): + attention_backend = "flashinfer" + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/spec/test_spec_standalone_extra.py b/test/registered/spec/test_spec_standalone_extra.py new file mode 100644 index 000000000..82272f741 --- /dev/null +++ b/test/registered/spec/test_spec_standalone_extra.py @@ -0,0 +1,34 @@ +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.server_fixtures.standalone_fixture import StandaloneServerBase +from sglang.test.test_utils import CustomTestCase + +# Non-V2 standalone speculative decoding tests (FA3, Triton, FlashInfer +# backends). Sibling V2 classes stay per-commit in test_spec_standalone.py. +register_cuda_ci(est_time=406, stage="extra-a", runner_config="1-gpu-large") + + +class TestStandaloneSpeculativeDecodingBase(StandaloneServerBase, CustomTestCase): + attention_backend = "fa3" + speculative_eagle_topk = 2 + speculative_num_draft_tokens = 7 + enable_spec_v2 = False + + +class TestStandaloneSpeculativeDecodingTriton(StandaloneServerBase, CustomTestCase): + attention_backend = "triton" + speculative_eagle_topk = 2 + speculative_num_draft_tokens = 7 + enable_spec_v2 = False + + +class TestStandaloneSpeculativeDecodingFlashinfer(StandaloneServerBase, CustomTestCase): + attention_backend = "flashinfer" + speculative_eagle_topk = 2 + speculative_num_draft_tokens = 7 + enable_spec_v2 = False + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/spec/test_standalone_speculative_decoding.py b/test/registered/spec/test_standalone_speculative_decoding.py deleted file mode 100644 index 8361ce8a7..000000000 --- a/test/registered/spec/test_standalone_speculative_decoding.py +++ /dev/null @@ -1,223 +0,0 @@ -import unittest -from types import SimpleNamespace - -import requests - -from sglang.srt.environ import envs -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_DRAFT_MODEL_STANDALONE, - DEFAULT_TARGET_MODEL_STANDALONE, - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) - -# Standalone speculative decoding tests (FA3, Triton, FlashInfer backends) -register_cuda_ci(est_time=406, stage="stage-b", runner_config="1-gpu-large") - -GSM_DATASET_PATH = None - -# Default server arguments shared across all tests -DEFAULT_SERVER_ARGS = [ - "--trust-remote-code", - "--cuda-graph-max-bs", - "8", - "--speculative-algorithm", - "STANDALONE", - "--speculative-draft-model-path", - DEFAULT_DRAFT_MODEL_STANDALONE, - "--speculative-num-steps", - "4", - "--speculative-eagle-topk", - "2", - "--speculative-num-draft-tokens", - "7", - "--mem-fraction-static", - 0.7, -] - -# Default server arguments for V2 tests -DEFAULT_SERVER_ARGS_V2 = [ - "--trust-remote-code", - "--cuda-graph-max-bs", - "8", - "--speculative-algorithm", - "STANDALONE", - "--speculative-draft-model-path", - DEFAULT_DRAFT_MODEL_STANDALONE, - "--speculative-num-steps", - "4", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "5", - "--mem-fraction-static", - 0.7, -] - - -class TestStandaloneSpeculativeDecodingBase(CustomTestCase): - - model = DEFAULT_TARGET_MODEL_STANDALONE - draft_model = DEFAULT_DRAFT_MODEL_STANDALONE - base_url = DEFAULT_URL_FOR_TEST - accuracy_threshold = 0.69 # derived tests need to override this - spec_decode_threshold = 3.6 # derived spec decoding tests need to override this - - @classmethod - def get_server_args(cls): - """Return the arguments for the server launch. Override in subclasses.""" - return DEFAULT_SERVER_ARGS + ["--attention-backend", "fa3"] - - @classmethod - def setUpClass(cls): - # disable deep gemm precompile to make launch server faster - # please don't do this if you want to make your inference workload faster - envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False) - envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False) - envs.SGLANG_ENABLE_SPEC_V2.set(False) - model = cls.model - cls.process = popen_launch_server( - model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=cls.get_server_args(), - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - envs.SGLANG_ENABLE_SPEC_V2.clear() - - def test_gsm8k(self): - requests.get(self.base_url + "/flush_cache") - - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=100, - num_threads=128, - num_shots=4, - gsm8k_data_path=GSM_DATASET_PATH, - ) - metrics = run_eval(args) - print(f"{metrics=}") - - # Use the appropriate metric key based on the test class - metric_key = "score" - self.assertGreaterEqual(metrics[metric_key], self.accuracy_threshold) - - server_info = requests.get(self.base_url + "/server_info") - avg_spec_accept_length = server_info.json()["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold) - - -class TestStandaloneV2SpeculativeDecodingBase(CustomTestCase): - - model = DEFAULT_TARGET_MODEL_STANDALONE - draft_model = DEFAULT_DRAFT_MODEL_STANDALONE - base_url = DEFAULT_URL_FOR_TEST - accuracy_threshold = 0.69 # derived tests need to override this - spec_decode_threshold = 3.6 # derived spec decoding tests need to override this - - @classmethod - def get_server_args(cls): - """Return the arguments for the server launch. Override in subclasses.""" - return DEFAULT_SERVER_ARGS_V2 + ["--attention-backend", "fa3"] - - @classmethod - def setUpClass(cls): - # disable deep gemm precompile to make launch server faster - # please don't do this if you want to make your inference workload faster - envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False) - envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False) - model = cls.model - cls.process = popen_launch_server( - model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=cls.get_server_args(), - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - requests.get(self.base_url + "/flush_cache") - - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=100, - num_threads=128, - num_shots=4, - gsm8k_data_path=GSM_DATASET_PATH, - ) - metrics = run_eval(args) - print(f"{metrics=}") - - # Use the appropriate metric key based on the test class - metric_key = "score" - self.assertGreaterEqual(metrics[metric_key], self.accuracy_threshold) - - server_info = requests.get(self.base_url + "/server_info") - avg_spec_accept_length = server_info.json()["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold) - - -class TestStandaloneSpeculativeDecodingTriton(TestStandaloneSpeculativeDecodingBase): - - @classmethod - def get_server_args(cls): - return DEFAULT_SERVER_ARGS + ["--attention-backend", "triton"] - - -class TestStandaloneSpeculativeDecodingFlashinfer( - TestStandaloneSpeculativeDecodingBase -): - @classmethod - def get_server_args(cls): - return DEFAULT_SERVER_ARGS + ["--attention-backend", "flashinfer"] - - -class TestStandaloneV2SpeculativeDecodingTriton( - TestStandaloneV2SpeculativeDecodingBase -): - - @classmethod - def get_server_args(cls): - return DEFAULT_SERVER_ARGS_V2 + ["--attention-backend", "triton"] - - def test_radix_attention(self): - run_radix_attention_test(self.base_url) - assert self.process.poll() is None - - -class TestStandaloneV2SpeculativeDecodingFlashinfer( - TestStandaloneV2SpeculativeDecodingBase -): - @classmethod - def get_server_args(cls): - return DEFAULT_SERVER_ARGS_V2 + ["--attention-backend", "flashinfer"] - - -if __name__ == "__main__": - unittest.main() diff --git a/test/run_suite.py b/test/run_suite.py index eb3ebc938..d1afb4e59 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -55,9 +55,17 @@ PER_COMMIT_SUITES = { "stage-c-test-8-gpu-h200", "stage-c-test-8-gpu-b200", "stage-c-test-deepep-4-gpu-h100", - "stage-c-test-deepep-8-gpu-h200", "stage-c-test-dsv4-4-gpu-b200", "stage-c-test-dsv4-8-gpu-h200", + # extra-a / extra-b: label-gated PR opt-in suites in pr-test-extra.yml + # (tests still tagged per-commit but skipped on default PR runs). + "extra-a-test-1-gpu-small", + "extra-a-test-1-gpu-large", + "extra-a-test-2-gpu-large", + "extra-b-test-4-gpu-h100", + "extra-b-test-4-gpu-b200", + "extra-b-test-8-gpu-h200", + "extra-b-test-deepep-8-gpu-h200", ], HWBackend.NPU: [ "stage-a-test-1-gpu-small",