[misc] Rename minilb install env & remove files & fix lint (#13831)
This commit is contained in:
@@ -1,6 +0,0 @@
|
|||||||
raise RuntimeError(
|
|
||||||
"""The 'mini_lb' module has been relocated to the 'sglang_router' package.
|
|
||||||
We recommend installing 'sglang-router' with Rust support for optimal performance.
|
|
||||||
If you encounter issues building the router with Rust, set the environment variable
|
|
||||||
'SGLANG_ROUTER_BUILD_NO_RUST=1' and add '--mini-lb' to the command line to use the Python version of 'mini_lb'."""
|
|
||||||
)
|
|
||||||
@@ -1,11 +1,13 @@
|
|||||||
import os
|
import os
|
||||||
|
import warnings
|
||||||
|
|
||||||
from setuptools import setup
|
from setuptools import setup
|
||||||
|
|
||||||
no_rust = os.environ.get("SGLANG_ROUTER_BUILD_NO_RUST") == "1"
|
with_rust = os.environ.get("SGLANG_ROUTER_BUILD_WITH_RUST", None)
|
||||||
|
with_rust = with_rust is None or (not with_rust.lower() in ["0", "false", "no"])
|
||||||
|
|
||||||
rust_extensions = []
|
rust_extensions = []
|
||||||
if not no_rust:
|
if with_rust:
|
||||||
from setuptools_rust import Binding, RustExtension
|
from setuptools_rust import Binding, RustExtension
|
||||||
|
|
||||||
rust_extensions.append(
|
rust_extensions.append(
|
||||||
@@ -15,6 +17,10 @@ if not no_rust:
|
|||||||
binding=Binding.PyO3,
|
binding=Binding.PyO3,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
|
warnings.warn(
|
||||||
|
"Building 'sglang-router' without Rust support. Performance may be degraded."
|
||||||
|
)
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
rust_extensions=rust_extensions,
|
rust_extensions=rust_extensions,
|
||||||
|
|||||||
@@ -291,35 +291,35 @@ class TestPrioritySchedulingMultipleRunningRequests(CustomTestCase):
|
|||||||
def test_priority_scheduling_preemption_token_offset_calculation(self):
|
def test_priority_scheduling_preemption_token_offset_calculation(self):
|
||||||
"""
|
"""
|
||||||
Verify correct token offset calculation during preemption.
|
Verify correct token offset calculation during preemption.
|
||||||
|
|
||||||
This test specifically targets the bug where rem_total_token_offset was incorrectly
|
This test specifically targets the bug where rem_total_token_offset was incorrectly
|
||||||
calculated using the incoming request's tokens instead of the preempted request's tokens
|
calculated using the incoming request's tokens instead of the preempted request's tokens
|
||||||
(related to issue #13111 and PR #13201).
|
(related to issue #13111 and PR #13201).
|
||||||
|
|
||||||
THE BUG:
|
THE BUG:
|
||||||
In schedule_policy.py line 700, the code was using:
|
In schedule_policy.py line 700, the code was using:
|
||||||
self.rem_total_token_offset -= self._get_running_request_total_token_offset(req)
|
self.rem_total_token_offset -= self._get_running_request_total_token_offset(req)
|
||||||
Instead of:
|
Instead of:
|
||||||
self.rem_total_token_offset -= self._get_running_request_total_token_offset(running_req)
|
self.rem_total_token_offset -= self._get_running_request_total_token_offset(running_req)
|
||||||
|
|
||||||
WHY THIS TEST CATCHES THE BUG:
|
WHY THIS TEST CATCHES THE BUG:
|
||||||
- Request 1 (preempted): 8000 tokens - This is what SHOULD be freed
|
- Request 1 (preempted): 8000 tokens - This is what SHOULD be freed
|
||||||
- Request 3 (incoming): 1000 tokens - This is what WAS freed (bug)
|
- Request 3 (incoming): 1000 tokens - This is what WAS freed (bug)
|
||||||
- Token difference: 8000 - 1000 = 7000 tokens incorrectly accounted
|
- Token difference: 8000 - 1000 = 7000 tokens incorrectly accounted
|
||||||
|
|
||||||
With the bug, the system thinks it only freed 1000 tokens instead of 8000 tokens.
|
With the bug, the system thinks it only freed 1000 tokens instead of 8000 tokens.
|
||||||
This causes incorrect memory accounting and can lead to:
|
This causes incorrect memory accounting and can lead to:
|
||||||
1. Scheduler believes less memory is available than actually is
|
1. Scheduler believes less memory is available than actually is
|
||||||
2. Subsequent requests (like Request 4) may fail to schedule or cause issues
|
2. Subsequent requests (like Request 4) may fail to schedule or cause issues
|
||||||
3. Memory calculations become increasingly inaccurate with each preemption
|
3. Memory calculations become increasingly inaccurate with each preemption
|
||||||
|
|
||||||
The test creates a scenario where:
|
The test creates a scenario where:
|
||||||
1. A low-priority request with many tokens (8000) starts running
|
1. A low-priority request with many tokens (8000) starts running
|
||||||
2. A high-priority request with few tokens (1000) arrives and triggers preemption
|
2. A high-priority request with few tokens (1000) arrives and triggers preemption
|
||||||
3. The system must correctly free 8000 tokens from the preempted request
|
3. The system must correctly free 8000 tokens from the preempted request
|
||||||
4. Additional requests can be scheduled only if tokens were correctly freed
|
4. Additional requests can be scheduled only if tokens were correctly freed
|
||||||
5. Execution order validates priority-based scheduling works correctly
|
5. Execution order validates priority-based scheduling works correctly
|
||||||
|
|
||||||
The large token difference (8x) makes the bug's impact obvious and testable.
|
The large token difference (8x) makes the bug's impact obvious and testable.
|
||||||
"""
|
"""
|
||||||
responses = asyncio.run(
|
responses = asyncio.run(
|
||||||
@@ -360,7 +360,7 @@ class TestPrioritySchedulingMultipleRunningRequests(CustomTestCase):
|
|||||||
_verify_genereate_responses(
|
_verify_genereate_responses(
|
||||||
responses, expected_status_and_error_messages, e2e_latencies
|
responses, expected_status_and_error_messages, e2e_latencies
|
||||||
)
|
)
|
||||||
|
|
||||||
# Verify execution order: high priority requests finish before low priority ones
|
# Verify execution order: high priority requests finish before low priority ones
|
||||||
# Request 3 (priority 100) should finish first
|
# Request 3 (priority 100) should finish first
|
||||||
# Request 4 (priority 50) should finish second
|
# Request 4 (priority 50) should finish second
|
||||||
|
|||||||
Reference in New Issue
Block a user