minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

tiny remove deprecated endpoint call (#13607)

Browse Source
This commit is contained in:
b8zhong
2025-12-05 09:54:49 -08:00
committed by GitHub
parent 1569fc7f49
commit ec7b2c16d9
36 changed files with 59 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/advanced_features/router.md
+1 -1
View File
@@ -45,7 +45,7 @@ SGLang Model Gateway is a high-performance model-routing gateway for large-scale
## Architecture ## Architecture
### Control Plane ### Control Plane
- **Worker Manager** discovers capabilities (`/get_server_info`, `/get_model_info`), tracks load, and registers/removes workers in the shared registry. - **Worker Manager** discovers capabilities (`/server_info`, `/model_info`), tracks load, and registers/removes workers in the shared registry.
- **Job Queue** serializes add/remove requests and exposes status (`/workers/{url}`) so clients can track onboarding progress. - **Job Queue** serializes add/remove requests and exposes status (`/workers/{url}`) so clients can track onboarding progress.
- **Load Monitor** feeds cache-aware and power-of-two policies with live worker load statistics. - **Load Monitor** feeds cache-aware and power-of-two policies with live worker load statistics.
- **Health Checker** continuously probes workers and updates readiness, circuit breaker state, and router metrics. - **Health Checker** continuously probes workers and updates readiness, circuit breaker state, and router metrics.
docs/advanced_features/server_arguments.md
+1 -1
View File
@@ -219,7 +219,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s
| Argument | Description | Defaults | Options | | Argument | Description | Defaults | Options |
| --- | --- | --- | --- | | --- | --- | --- | --- |
| `--json-model-override-args` | A dictionary in JSON string format used to override default model configurations. | `{}` | Type: str | | `--json-model-override-args` | A dictionary in JSON string format used to override default model configurations. | `{}` | Type: str |
| `--preferred-sampling-params` | json-formatted sampling settings that will be returned in /get_model_info | `None` | Type: str | | `--preferred-sampling-params` | json-formatted sampling settings that will be returned in /model_info | `None` | Type: str |
## LoRA ## LoRA
| Argument | Description | Defaults | Options | | Argument | Description | Defaults | Options |
docs/basic_usage/native_api.ipynb
+5 -5
View File
@@ -9,8 +9,8 @@
"Apart from the OpenAI compatible APIs, the SGLang Runtime also provides its native server APIs. We introduce the following APIs:\n", "Apart from the OpenAI compatible APIs, the SGLang Runtime also provides its native server APIs. We introduce the following APIs:\n",
"\n", "\n",
"- `/generate` (text generation model)\n", "- `/generate` (text generation model)\n",
"- `/get_model_info`\n", "- `/model_info`\n",
"- `/get_server_info`\n", "- `/server_info`\n",
"- `/health`\n", "- `/health`\n",
"- `/health_generate`\n", "- `/health_generate`\n",
"- `/flush_cache`\n", "- `/flush_cache`\n",
@@ -97,7 +97,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"url = f\"http://localhost:{port}/get_model_info\"\n", "url = f\"http://localhost:{port}/model_info\"\n",
"\n", "\n",
"response = requests.get(url)\n", "response = requests.get(url)\n",
"response_json = response.json()\n", "response_json = response.json()\n",
@@ -123,7 +123,7 @@
"source": [ "source": [
"## Get Server Info\n", "## Get Server Info\n",
"Gets the server information including CLI arguments, token limits, and memory pool sizes.\n", "Gets the server information including CLI arguments, token limits, and memory pool sizes.\n",
"- Note: `get_server_info` merges the following deprecated endpoints:\n", "- Note: `server_info` merges the following deprecated endpoints:\n",
" - `get_server_args`\n", " - `get_server_args`\n",
" - `get_memory_pool_size` \n", " - `get_memory_pool_size` \n",
" - `get_max_total_num_tokens`" " - `get_max_total_num_tokens`"
@@ -135,7 +135,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"url = f\"http://localhost:{port}/get_server_info\"\n", "url = f\"http://localhost:{port}/server_info\"\n",
"\n", "\n",
"response = requests.get(url)\n", "response = requests.get(url)\n",
"print_highlight(response.text)" "print_highlight(response.text)"
docs/developer_guide/bench_serving.md
+1 -1
View File
@@ -352,4 +352,4 @@ python3 -m sglang.bench_serving \
### Notes ### Notes
- The script raises the file descriptor soft limit (`RLIMIT_NOFILE`) to help with many concurrent connections. - The script raises the file descriptor soft limit (`RLIMIT_NOFILE`) to help with many concurrent connections.
- For sglang, `/get_server_info` is queried post-run to report speculative decoding accept length when available. - For sglang, `/server_info` is queried post-run to report speculative decoding accept length when available.
python/sglang/bench_one_batch_server.py
+2 -2
View File
@@ -339,7 +339,7 @@ def run_one_case(
output_throughput = batch_size * output_len / (latency - last_ttft) output_throughput = batch_size * output_len / (latency - last_ttft)
overall_throughput = batch_size * (input_len + output_len) / latency overall_throughput = batch_size * (input_len + output_len) / latency
server_info = requests.get(url + "/get_server_info").json() server_info = requests.get(url + "/server_info").json()
internal_state = server_info.get("internal_states", [{}]) internal_state = server_info.get("internal_states", [{}])
last_gen_throughput = internal_state[0].get("last_gen_throughput", None) or -1 last_gen_throughput = internal_state[0].get("last_gen_throughput", None) or -1
acc_length = internal_state[0].get("avg_spec_accept_length", None) or -1 acc_length = internal_state[0].get("avg_spec_accept_length", None) or -1
@@ -451,7 +451,7 @@ def run_benchmark(server_args: ServerArgs, bench_args: BenchArgs):
proc, base_url = launch_server_process(server_args) proc, base_url = launch_server_process(server_args)
# Get tokenizer # Get tokenizer
server_info = requests.get(base_url + "/get_server_info").json() server_info = requests.get(base_url + "/server_info").json()
if "tokenizer_path" in server_info: if "tokenizer_path" in server_info:
tokenizer_path = server_info["tokenizer_path"] tokenizer_path = server_info["tokenizer_path"]
elif "prefill" in server_info: elif "prefill" in server_info:
python/sglang/bench_serving.py
+2 -2
View File
@@ -2062,7 +2062,7 @@ async def benchmark(
if "sglang" in backend: if "sglang" in backend:
server_info = requests.get( server_info = requests.get(
base_url + "/get_server_info", headers=get_auth_headers() base_url + "/server_info", headers=get_auth_headers()
) )
if server_info.status_code == 200: if server_info.status_code == 200:
server_info_json = server_info.json() server_info_json = server_info.json()
@@ -2180,7 +2180,7 @@ async def benchmark(
print("{:<40} {:<10.2f}".format("Max ITL (ms):", metrics.max_itl_ms)) print("{:<40} {:<10.2f}".format("Max ITL (ms):", metrics.max_itl_ms))
print("=" * 50) print("=" * 50)
resp = requests.get(base_url + "/get_server_info", headers=get_auth_headers()) resp = requests.get(base_url + "/server_info", headers=get_auth_headers())
server_info = resp.json() if resp.status_code == 200 else None server_info = resp.json() if resp.status_code == 200 else None
if ( if (
python/sglang/lang/backend/runtime_endpoint.py
+3 -3
View File
@@ -38,7 +38,7 @@ class RuntimeEndpoint(BaseBackend):
self.verify = verify self.verify = verify
res = http_request( res = http_request(
self.base_url + "/get_model_info", self.base_url + "/model_info",
api_key=self.api_key, api_key=self.api_key,
verify=self.verify, verify=self.verify,
) )
@@ -66,7 +66,7 @@ class RuntimeEndpoint(BaseBackend):
def get_server_info(self): def get_server_info(self):
res = http_request( res = http_request(
self.base_url + "/get_server_info", self.base_url + "/server_info",
api_key=self.api_key, api_key=self.api_key,
verify=self.verify, verify=self.verify,
) )
@@ -514,7 +514,7 @@ class Runtime:
async def get_server_info(self): async def get_server_info(self):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
async with session.get(f"{self.url}/get_server_info") as response: async with session.get(f"{self.url}/server_info") as response:
if response.status == 200: if response.status == 200:
return await response.json() return await response.json()
else: else:
python/sglang/profiler.py
+1 -1
View File
@@ -41,7 +41,7 @@ def run_profile(
# Dump server args. # Dump server args.
file_path = Path(output_dir) / "server_args.json" file_path = Path(output_dir) / "server_args.json"
if not file_path.exists(): if not file_path.exists():
response = requests.get(url + "/get_server_info") response = requests.get(url + "/server_info")
response.raise_for_status() response.raise_for_status()
server_args_data = response.json() server_args_data = response.json()
with open(file_path, "w") as file: with open(file_path, "w") as file:
python/sglang/srt/entrypoints/http_server.py
+1 -1
View File
@@ -1480,7 +1480,7 @@ def _execute_server_warmup(
for _ in range(120): for _ in range(120):
time.sleep(1) time.sleep(1)
try: try:
res = requests.get(url + "/get_model_info", timeout=5, headers=headers) res = requests.get(url + "/model_info", timeout=5, headers=headers)
assert res.status_code == 200, f"{res=}, {res.text=}" assert res.status_code == 200, f"{res=}, {res.text=}"
success = True success = True
break break
python/sglang/srt/server_args.py
+1 -1
View File
@@ -2797,7 +2797,7 @@ class ServerArgs:
parser.add_argument( parser.add_argument(
"--preferred-sampling-params", "--preferred-sampling-params",
type=str, type=str,
help="json-formatted sampling settings that will be returned in /get_model_info", help="json-formatted sampling settings that will be returned in /model_info",
) )
# LoRA # LoRA
scripts/playground/bench_speculative.py
+1 -1
View File
@@ -120,7 +120,7 @@ def send_one_batch(base_url, num_prompts, batch_size, processor, is_multimodal):
acc_length = results["accept_length"] or 1.0 acc_length = results["accept_length"] or 1.0
avg_output_token = results["total_output_tokens"] / results["completed"] avg_output_token = results["total_output_tokens"] / results["completed"]
server_info = requests.get(base_url + "/get_server_info").json() server_info = requests.get(base_url + "/server_info").json()
# We use 20% percentile instead of median on purpose # We use 20% percentile instead of median on purpose
step_time = np.percentile( step_time = np.percentile(
server_info["internal_states"][0]["step_time_dict"][str(batch_size)], 20 server_info["internal_states"][0]["step_time_dict"][str(batch_size)], 20
test/manual/test_deepseek_v32_cp_single_node.py
+1 -1
View File
@@ -79,7 +79,7 @@ class TestDeepseekV32CP(CustomTestCase):
metrics = run_eval_few_shot_gsm8k(args) metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}") print(f"{metrics=}")
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]
test/manual/test_eagle_infer_beta_dp_attention.py
+1 -1
View File
@@ -77,7 +77,7 @@ class TestEagleDPAttnServerBase(CustomTestCase):
metrics = run_eval_few_shot_gsm8k(args) metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}") print(f"{metrics=}")
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]
test/manual/test_weight_version.py
+10 -10
View File
@@ -3,7 +3,7 @@ Test weight version functionality.
This test suite verifies the weight_version feature implementation including: This test suite verifies the weight_version feature implementation including:
1. Default weight_version setting 1. Default weight_version setting
2. /get_weight_version endpoint 2. /weight_version endpoint
3. /update_weight_version endpoint 3. /update_weight_version endpoint
4. /generate request meta_info contains weight_version 4. /generate request meta_info contains weight_version
5. OpenAI API response metadata contains weight_version 5. OpenAI API response metadata contains weight_version
@@ -48,13 +48,13 @@ class TestWeightVersion(CustomTestCase):
def test_weight_version_comprehensive(self): def test_weight_version_comprehensive(self):
"""Comprehensive test for all weight_version functionality.""" """Comprehensive test for all weight_version functionality."""
response = requests.get(f"{self.base_url}/get_model_info") response = requests.get(f"{self.base_url}/model_info")
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
data = response.json() data = response.json()
self.assertIn("weight_version", data) self.assertIn("weight_version", data)
self.assertEqual(data["weight_version"], "test_version_1.0") self.assertEqual(data["weight_version"], "test_version_1.0")
response = requests.get(f"{self.base_url}/get_weight_version") response = requests.get(f"{self.base_url}/weight_version")
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
data = response.json() data = response.json()
self.assertIn("weight_version", data) self.assertIn("weight_version", data)
@@ -114,7 +114,7 @@ class TestWeightVersion(CustomTestCase):
self.assertTrue(data["success"]) self.assertTrue(data["success"])
self.assertEqual(data["new_version"], "updated_version_2.0") self.assertEqual(data["new_version"], "updated_version_2.0")
response = requests.get(f"{self.base_url}/get_weight_version") response = requests.get(f"{self.base_url}/weight_version")
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
data = response.json() data = response.json()
self.assertEqual(data["weight_version"], "updated_version_2.0") self.assertEqual(data["weight_version"], "updated_version_2.0")
@@ -148,13 +148,13 @@ class TestWeightVersion(CustomTestCase):
self.assertTrue(data["success"]) self.assertTrue(data["success"])
self.assertEqual(data["new_version"], "final_version_3.0") self.assertEqual(data["new_version"], "final_version_3.0")
# Check /get_weight_version # Check /weight_version
response = requests.get(f"{self.base_url}/get_weight_version") response = requests.get(f"{self.base_url}/weight_version")
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
self.assertEqual(response.json()["weight_version"], "final_version_3.0") self.assertEqual(response.json()["weight_version"], "final_version_3.0")
# Check /get_model_info # Check /model_info
response = requests.get(f"{self.base_url}/get_model_info") response = requests.get(f"{self.base_url}/model_info")
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
self.assertEqual(response.json()["weight_version"], "final_version_3.0") self.assertEqual(response.json()["weight_version"], "final_version_3.0")
@@ -193,7 +193,7 @@ class TestWeightVersion(CustomTestCase):
print("Testing weight_version update with real weight operations...") print("Testing weight_version update with real weight operations...")
# Get current model info for reference # Get current model info for reference
model_info_response = requests.get(f"{self.base_url}/get_model_info") model_info_response = requests.get(f"{self.base_url}/model_info")
self.assertEqual(model_info_response.status_code, 200) self.assertEqual(model_info_response.status_code, 200)
current_model_path = model_info_response.json()["model_path"] current_model_path = model_info_response.json()["model_path"]
@@ -214,7 +214,7 @@ class TestWeightVersion(CustomTestCase):
) )
# Verify version was updated # Verify version was updated
version_response = requests.get(f"{self.base_url}/get_weight_version") version_response = requests.get(f"{self.base_url}/weight_version")
self.assertEqual(version_response.status_code, 200) self.assertEqual(version_response.status_code, 200)
self.assertEqual( self.assertEqual(
version_response.json()["weight_version"], "disk_update_v2.0.0" version_response.json()["weight_version"], "disk_update_v2.0.0"
test/srt/ep/test_deepep_large.py
+1 -1
View File
@@ -143,7 +143,7 @@ class TestDeepseekMTP(CustomTestCase):
self.assertGreater(metrics["accuracy"], 0.92) self.assertGreater(metrics["accuracy"], 0.92)
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]
test/srt/ep/test_deepep_small.py
+2 -2
View File
@@ -307,7 +307,7 @@ class TestMTP(CustomTestCase):
self.assertGreater(metrics["accuracy"], 0.60) self.assertGreater(metrics["accuracy"], 0.60)
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]
@@ -381,7 +381,7 @@ class TestMTPWithTBO(CustomTestCase):
self.assertGreater(metrics["accuracy"], 0.60) self.assertGreater(metrics["accuracy"], 0.60)
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]
test/srt/hicache/test_hicache_variants.py
+1 -1
View File
@@ -152,7 +152,7 @@ class TestHiCacheEagle(HiCacheBaseServer, HiCacheEvalMixin):
self.assertGreaterEqual(metrics["score"], self.expected_mmlu_score) self.assertGreaterEqual(metrics["score"], self.expected_mmlu_score)
# EAGLE-specific check # EAGLE-specific check
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
print(f"{server_info=}") print(f"{server_info=}")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
test/srt/quant/test_w4a8_deepseek_v3.py
+1 -1
View File
@@ -103,7 +103,7 @@ class TestDeepseekV3W4Afp8Mtp(CustomTestCase):
metrics = run_eval_few_shot_gsm8k(args) metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}") print(f"{metrics=}")
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]
test/srt/rl/test_update_weights_from_disk.py
+3 -3
View File
@@ -113,7 +113,7 @@ class TestServerUpdateWeightsFromDisk(CustomTestCase):
return response.json() return response.json()
def get_model_info(self): def get_model_info(self):
response = requests.get(self.base_url + "/get_model_info") response = requests.get(self.base_url + "/model_info")
model_path = response.json()["model_path"] model_path = response.json()["model_path"]
print(json.dumps(response.json())) print(json.dumps(response.json()))
return model_path return model_path
@@ -254,7 +254,7 @@ class TestServerUpdateWeightsFromDiskAbortAllRequests(CustomTestCase):
return response.json() return response.json()
def get_model_info(self): def get_model_info(self):
response = requests.get(self.base_url + "/get_model_info") response = requests.get(self.base_url + "/model_info")
model_path = response.json()["model_path"] model_path = response.json()["model_path"]
print(json.dumps(response.json())) print(json.dumps(response.json()))
return model_path return model_path
@@ -386,7 +386,7 @@ class TestUpdateWeightsFromDiskParameterized(CustomTestCase):
return response.json()["text"] return response.json()["text"]
def get_model_info(): def get_model_info():
response = requests.get(base_url + "/get_model_info") response = requests.get(base_url + "/model_info")
model_path = response.json()["model_path"] model_path = response.json()["model_path"]
print(json.dumps(response.json())) print(json.dumps(response.json()))
return model_path return model_path
test/srt/rl/test_update_weights_from_tensor.py
+1 -1
View File
@@ -209,7 +209,7 @@ class TestServerUpdateWeightsFromTensorNonBlocking(CustomTestCase):
return response.json() return response.json()
def get_model_info(self): def get_model_info(self):
response = requests.get(self.base_url + "/get_model_info") response = requests.get(self.base_url + "/model_info")
model_path = response.json()["model_path"] model_path = response.json()["model_path"]
print(json.dumps(response.json())) print(json.dumps(response.json()))
return model_path return model_path
test/srt/test_data_parallelism.py
+2 -2
View File
@@ -65,12 +65,12 @@ class TestDataParallelism(CustomTestCase):
def test_get_memory_pool_size(self): def test_get_memory_pool_size(self):
# use `get_server_info` instead since `get_memory_pool_size` is merged into `get_server_info` # use `get_server_info` instead since `get_memory_pool_size` is merged into `get_server_info`
response = requests.get(self.base_url + "/get_server_info") response = requests.get(self.base_url + "/server_info")
assert response.status_code == 200 assert response.status_code == 200
time.sleep(1) time.sleep(1)
response = requests.get(self.base_url + "/get_server_info") response = requests.get(self.base_url + "/server_info")
assert response.status_code == 200 assert response.status_code == 200
test/srt/test_deepseek_v32_mtp.py
+1 -1
View File
@@ -69,7 +69,7 @@ class TestDeepseekV32MTP(CustomTestCase):
metrics = run_eval_few_shot_gsm8k(args) metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}") print(f"{metrics=}")
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]
test/srt/test_deepseek_v3_fp4_4gpu.py
+1 -1
View File
@@ -143,7 +143,7 @@ class TestDeepseekV3FP4MTP(CustomTestCase):
metrics = run_eval_few_shot_gsm8k(args) metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}") print(f"{metrics=}")
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]
test/srt/test_deepseek_v3_mtp.py
+1 -1
View File
@@ -67,7 +67,7 @@ class TestDeepseekV3MTP(CustomTestCase):
metrics = run_eval_few_shot_gsm8k(args) metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}") print(f"{metrics=}")
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]
test/srt/test_dp_attention.py
+1 -1
View File
@@ -112,7 +112,7 @@ class TestDPAttentionDP2TP2DeepseekV3MTP(CustomTestCase):
self.assertGreater(metrics["accuracy"], 0.60) self.assertGreater(metrics["accuracy"], 0.60)
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]
test/srt/test_eagle_dp_attention.py
+1 -1
View File
@@ -78,7 +78,7 @@ class TestEAGLE3EngineDPAttention(CustomTestCase):
metrics = run_eval_few_shot_gsm8k(args) metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}") print(f"{metrics=}")
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
server_data = server_info.json() server_data = server_info.json()
# Try to get avg_spec_accept_length # Try to get avg_spec_accept_length
test/srt/test_eagle_infer_b.py
+1 -1
View File
@@ -144,7 +144,7 @@ class TestEAGLEServer(CustomTestCase):
print(f"{metrics=}") print(f"{metrics=}")
self.assertGreater(metrics["accuracy"], 0.20) self.assertGreater(metrics["accuracy"], 0.20)
server_info = requests.get(self.base_url + "/get_server_info").json() server_info = requests.get(self.base_url + "/server_info").json()
avg_spec_accept_length = server_info["internal_states"][0][ avg_spec_accept_length = server_info["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]
test/srt/test_fa3.py
+1 -1
View File
@@ -112,7 +112,7 @@ class BaseFlashAttentionTest(CustomTestCase):
self.assertGreater(metrics[metric_key], self.accuracy_threshold) self.assertGreater(metrics[metric_key], self.accuracy_threshold)
if self.speculative_decode: if self.speculative_decode:
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]
test/srt/test_flashmla.py
+1 -1
View File
@@ -119,7 +119,7 @@ class TestFlashMLAMTP(CustomTestCase):
self.assertGreater(metrics["accuracy"], 0.60) self.assertGreater(metrics["accuracy"], 0.60)
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
print(f"{server_info=}") print(f"{server_info=}")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
test/srt/test_hybrid_attn_backend.py
+1 -1
View File
@@ -88,7 +88,7 @@ class TestHybridAttnBackendBase(CustomTestCase):
self.assertGreater(metrics[metric_key], self.accuracy_threshold) self.assertGreater(metrics[metric_key], self.accuracy_threshold)
if self.speculative_decode: if self.speculative_decode:
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]
test/srt/test_mla_deepseek_v3.py
+1 -1
View File
@@ -179,7 +179,7 @@ class TestDeepseekV3MTP(CustomTestCase):
self.assertGreater(metrics["accuracy"], 0.60) self.assertGreater(metrics["accuracy"], 0.60)
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]
test/srt/test_mla_flashinfer.py
+1 -1
View File
@@ -111,7 +111,7 @@ class TestFlashinferMLAMTP(CustomTestCase):
self.assertGreater(metrics["accuracy"], 0.60) self.assertGreater(metrics["accuracy"], 0.60)
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
print(f"{server_info=}") print(f"{server_info=}")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
test/srt/test_mla_int8_deepseek_v3.py
+2 -2
View File
@@ -113,7 +113,7 @@ class TestDeepseekV3MTPChannelInt8(CustomTestCase):
self.assertGreater(metrics["accuracy"], 0.60) self.assertGreater(metrics["accuracy"], 0.60)
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]
@@ -217,7 +217,7 @@ class TestDeepseekV3MTPBlockInt8(CustomTestCase):
self.assertGreater(metrics["accuracy"], 0.60) self.assertGreater(metrics["accuracy"], 0.60)
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]
test/srt/test_ngram_speculative_decoding.py
+1 -1
View File
@@ -80,7 +80,7 @@ class TestNgramSpeculativeDecodingBase(CustomTestCase):
metric_key = "accuracy" metric_key = "accuracy"
self.assertGreater(metrics[metric_key], self.accuracy_threshold) self.assertGreater(metrics[metric_key], self.accuracy_threshold)
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]
test/srt/test_srt_endpoint.py
+2 -2
View File
@@ -496,7 +496,7 @@ class TestSRTEndpoint(CustomTestCase):
self.assertEqual(send_and_check_cached_tokens(range(0, 11000)), 10000) self.assertEqual(send_and_check_cached_tokens(range(0, 11000)), 10000)
def test_get_server_info(self): def test_get_server_info(self):
response = requests.get(self.base_url + "/get_server_info") response = requests.get(self.base_url + "/server_info")
response_json = response.json() response_json = response.json()
max_total_num_tokens = response_json["max_total_num_tokens"] max_total_num_tokens = response_json["max_total_num_tokens"]
@@ -626,7 +626,7 @@ class TestSRTEndpoint(CustomTestCase):
tp = ThreadPoolExecutor(max_workers=30) tp = ThreadPoolExecutor(max_workers=30)
def s(): def s():
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
server_info.json() server_info.json()
futures = [] futures = []
test/srt/test_standalone_speculative_decoding.py
+1 -1
View File
@@ -88,7 +88,7 @@ class TestStandaloneSpeculativeDecodingBase(CustomTestCase):
metric_key = "accuracy" metric_key = "accuracy"
self.assertGreater(metrics[metric_key], self.accuracy_threshold) self.assertGreater(metrics[metric_key], self.accuracy_threshold)
server_info = requests.get(self.base_url + "/get_server_info") server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][ avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length" "avg_spec_accept_length"
] ]